263 return 0;
264}
265
266/**
267 * Select the right pool or requested caching state and ttm flags. */
268static struct ttm_page_pool *ttm_get_pool(int flags,
269 enum ttm_caching_state cstate)
270{
271 int pool_index;
272
273 if (cstate == tt_cached)
274 return NULL;
275
276 if (cstate == tt_wc)
277 pool_index = 0x0;
278 else
279 pool_index = 0x1;
280
281 if (flags & TTM_PAGE_FLAG_DMA32)
282 pool_index |= 0x2;
283
284 return &_manager->pools[pool_index];
285}
286
287/* set memory back to wb and free the pages. */
288static void ttm_pages_put(vm_page_t *pages, unsigned npages)
289{
290 unsigned i;
291
292 /* Our VM handles vm memattr automatically on the page free. */
293 if (set_pages_array_wb(pages, npages))
294 printf("[TTM] Failed to set %d pages to wb!\n", npages);
295 for (i = 0; i < npages; ++i)
296 ttm_vm_page_free(pages[i]);
297}
298
299static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
300 unsigned freed_pages)
301{
302 pool->npages -= freed_pages;
303 pool->nfrees += freed_pages;
304}
305
306/**
307 * Free pages from pool.
308 *
309 * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
310 * number of pages in one go.
311 *
312 * @pool: to free the pages from
313 * @free_all: If set to true will free all pages in pool
314 **/
315static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free)
316{
317 vm_page_t p, p1;
318 vm_page_t *pages_to_free;
319 unsigned freed_pages = 0,
320 npages_to_free = nr_free;
321 unsigned i;
322
323 if (NUM_PAGES_TO_ALLOC < nr_free)
324 npages_to_free = NUM_PAGES_TO_ALLOC;
325
326 pages_to_free = malloc(npages_to_free * sizeof(vm_page_t),
327 M_TEMP, M_WAITOK | M_ZERO);
328
329restart:
330 mtx_lock(&pool->lock);
331
332 TAILQ_FOREACH_REVERSE_SAFE(p, &pool->list, pglist, plinks.q, p1) {
333 if (freed_pages >= npages_to_free)
334 break;
335
336 pages_to_free[freed_pages++] = p;
337 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */
338 if (freed_pages >= NUM_PAGES_TO_ALLOC) {
339 /* remove range of pages from the pool */
340 for (i = 0; i < freed_pages; i++)
341 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q);
342
343 ttm_pool_update_free_locked(pool, freed_pages);
344 /**
345 * Because changing page caching is costly
346 * we unlock the pool to prevent stalling.
347 */
348 mtx_unlock(&pool->lock);
349
350 ttm_pages_put(pages_to_free, freed_pages);
351 if (likely(nr_free != FREE_ALL_PAGES))
352 nr_free -= freed_pages;
353
354 if (NUM_PAGES_TO_ALLOC >= nr_free)
355 npages_to_free = nr_free;
356 else
357 npages_to_free = NUM_PAGES_TO_ALLOC;
358
359 freed_pages = 0;
360
361 /* free all so restart the processing */
362 if (nr_free)
363 goto restart;
364
365 /* Not allowed to fall through or break because
366 * following context is inside spinlock while we are
367 * outside here.
368 */
369 goto out;
370
371 }
372 }
373
374 /* remove range of pages from the pool */
375 if (freed_pages) {
376 for (i = 0; i < freed_pages; i++)
377 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q);
378
379 ttm_pool_update_free_locked(pool, freed_pages);
380 nr_free -= freed_pages;
381 }
382
383 mtx_unlock(&pool->lock);
384
385 if (freed_pages)
386 ttm_pages_put(pages_to_free, freed_pages);
387out:
388 free(pages_to_free, M_TEMP);
389 return nr_free;
390}
391
392/* Get good estimation how many pages are free in pools */
393static int ttm_pool_get_num_unused_pages(void)
394{
395 unsigned i;
396 int total = 0;
397 for (i = 0; i < NUM_POOLS; ++i)
398 total += _manager->pools[i].npages;
399
400 return total;
401}
402
403/**
404 * Callback for mm to request pool to reduce number of page held.
405 */
406static int ttm_pool_mm_shrink(void *arg)
407{
408 static unsigned int start_pool = 0;
409 unsigned i;
410 unsigned pool_offset = atomic_fetchadd_int(&start_pool, 1);
411 struct ttm_page_pool *pool;
412 int shrink_pages = 100; /* XXXKIB */
413
414 pool_offset = pool_offset % NUM_POOLS;
415 /* select start pool in round robin fashion */
416 for (i = 0; i < NUM_POOLS; ++i) {
417 unsigned nr_free = shrink_pages;
418 if (shrink_pages == 0)
419 break;
420 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
421 shrink_pages = ttm_page_pool_free(pool, nr_free);
422 }
423 /* return estimated number of unused pages in pool */
424 return ttm_pool_get_num_unused_pages();
425}
426
427static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
428{
429
430 manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem,
431 ttm_pool_mm_shrink, manager, EVENTHANDLER_PRI_ANY);
432}
433
434static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
435{
436
437 EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler);
438}
439
440static int ttm_set_pages_caching(vm_page_t *pages,
441 enum ttm_caching_state cstate, unsigned cpages)
442{
443 int r = 0;
444 /* Set page caching */
445 switch (cstate) {
446 case tt_uncached:
447 r = set_pages_array_uc(pages, cpages);
448 if (r)
449 printf("[TTM] Failed to set %d pages to uc!\n", cpages);
450 break;
451 case tt_wc:
452 r = set_pages_array_wc(pages, cpages);
453 if (r)
454 printf("[TTM] Failed to set %d pages to wc!\n", cpages);
455 break;
456 default:
457 break;
458 }
459 return r;
460}
461
462/**
463 * Free pages the pages that failed to change the caching state. If there is
464 * any pages that have changed their caching state already put them to the
465 * pool.
466 */
467static void ttm_handle_caching_state_failure(struct pglist *pages,
468 int ttm_flags, enum ttm_caching_state cstate,
469 vm_page_t *failed_pages, unsigned cpages)
470{
471 unsigned i;
472 /* Failed pages have to be freed */
473 for (i = 0; i < cpages; ++i) {
474 TAILQ_REMOVE(pages, failed_pages[i], plinks.q);
475 ttm_vm_page_free(failed_pages[i]);
476 }
477}
478
479/**
480 * Allocate new pages with correct caching.
481 *
482 * This function is reentrant if caller updates count depending on number of
483 * pages returned in pages array.
484 */
485static int ttm_alloc_new_pages(struct pglist *pages, int ttm_alloc_flags,
486 int ttm_flags, enum ttm_caching_state cstate, unsigned count)
487{
488 vm_page_t *caching_array;
489 vm_page_t p;
490 int r = 0;
491 unsigned i, cpages, aflags;
492 unsigned max_cpages = min(count,
493 (unsigned)(PAGE_SIZE/sizeof(vm_page_t)));
494
495 aflags = VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ |
496 ((ttm_alloc_flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0 ?
497 VM_ALLOC_ZERO : 0);
498
499 /* allocate array for page caching change */
500 caching_array = malloc(max_cpages * sizeof(vm_page_t), M_TEMP,
501 M_WAITOK | M_ZERO);
502
503 for (i = 0, cpages = 0; i < count; ++i) {
504 p = vm_page_alloc_contig(NULL, 0, aflags, 1, 0,
505 (ttm_alloc_flags & TTM_PAGE_FLAG_DMA32) ? 0xffffffff :
506 VM_MAX_ADDRESS, PAGE_SIZE, 0,
507 ttm_caching_state_to_vm(cstate));
508 if (!p) {
509 printf("[TTM] Unable to get page %u\n", i);
510
511 /* store already allocated pages in the pool after
512 * setting the caching state */
513 if (cpages) {
514 r = ttm_set_pages_caching(caching_array,
515 cstate, cpages);
516 if (r)
517 ttm_handle_caching_state_failure(pages,
518 ttm_flags, cstate,
519 caching_array, cpages);
520 }
521 r = -ENOMEM;
522 goto out;
523 }
524 p->oflags &= ~VPO_UNMANAGED;
525 p->flags |= PG_FICTITIOUS;
526
527#ifdef CONFIG_HIGHMEM /* KIB: nop */
528 /* gfp flags of highmem page should never be dma32 so we
529 * we should be fine in such case
530 */
531 if (!PageHighMem(p))
532#endif
533 {
534 caching_array[cpages++] = p;
535 if (cpages == max_cpages) {
536
537 r = ttm_set_pages_caching(caching_array,
538 cstate, cpages);
539 if (r) {
540 ttm_handle_caching_state_failure(pages,
541 ttm_flags, cstate,
542 caching_array, cpages);
543 goto out;
544 }
545 cpages = 0;
546 }
547 }
548
549 TAILQ_INSERT_HEAD(pages, p, plinks.q);
550 }
551
552 if (cpages) {
553 r = ttm_set_pages_caching(caching_array, cstate, cpages);
554 if (r)
555 ttm_handle_caching_state_failure(pages,
556 ttm_flags, cstate,
557 caching_array, cpages);
558 }
559out:
560 free(caching_array, M_TEMP);
561
562 return r;
563}
564
565/**
566 * Fill the given pool if there aren't enough pages and the requested number of
567 * pages is small.
568 */
569static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
570 int ttm_flags, enum ttm_caching_state cstate, unsigned count)
571{
572 vm_page_t p;
573 int r;
574 unsigned cpages = 0;
575 /**
576 * Only allow one pool fill operation at a time.
577 * If pool doesn't have enough pages for the allocation new pages are
578 * allocated from outside of pool.
579 */
580 if (pool->fill_lock)
581 return;
582
583 pool->fill_lock = true;
584
585 /* If allocation request is small and there are not enough
586 * pages in a pool we fill the pool up first. */
587 if (count < _manager->options.small
588 && count > pool->npages) {
589 struct pglist new_pages;
590 unsigned alloc_size = _manager->options.alloc_size;
591
592 /**
593 * Can't change page caching if in irqsave context. We have to
594 * drop the pool->lock.
595 */
596 mtx_unlock(&pool->lock);
597
598 TAILQ_INIT(&new_pages);
599 r = ttm_alloc_new_pages(&new_pages, pool->ttm_page_alloc_flags,
600 ttm_flags, cstate, alloc_size);
601 mtx_lock(&pool->lock);
602
603 if (!r) {
604 TAILQ_CONCAT(&pool->list, &new_pages, plinks.q);
605 ++pool->nrefills;
606 pool->npages += alloc_size;
607 } else {
608 printf("[TTM] Failed to fill pool (%p)\n", pool);
609 /* If we have any pages left put them to the pool. */
610 TAILQ_FOREACH(p, &pool->list, plinks.q) {
611 ++cpages;
612 }
613 TAILQ_CONCAT(&pool->list, &new_pages, plinks.q);
614 pool->npages += cpages;
615 }
616
617 }
618 pool->fill_lock = false;
619}
620
621/**
622 * Cut 'count' number of pages from the pool and put them on the return list.
623 *
624 * @return count of pages still required to fulfill the request.
625 */
626static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
627 struct pglist *pages,
628 int ttm_flags,
629 enum ttm_caching_state cstate,
630 unsigned count)
631{
632 vm_page_t p;
633 unsigned i;
634
635 mtx_lock(&pool->lock);
636 ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count);
637
638 if (count >= pool->npages) {
639 /* take all pages from the pool */
640 TAILQ_CONCAT(pages, &pool->list, plinks.q);
641 count -= pool->npages;
642 pool->npages = 0;
643 goto out;
644 }
645 for (i = 0; i < count; i++) {
646 p = TAILQ_FIRST(&pool->list);
647 TAILQ_REMOVE(&pool->list, p, plinks.q);
648 TAILQ_INSERT_TAIL(pages, p, plinks.q);
649 }
650 pool->npages -= count;
651 count = 0;
652out:
653 mtx_unlock(&pool->lock);
654 return count;
655}
656
657/* Put all pages in pages list to correct pool to wait for reuse */
658static void ttm_put_pages(vm_page_t *pages, unsigned npages, int flags,
659 enum ttm_caching_state cstate)
660{
661 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
662 unsigned i;
663
664 if (pool == NULL) {
665 /* No pool for this memory type so free the pages */
666 for (i = 0; i < npages; i++) {
667 if (pages[i]) {
668 ttm_vm_page_free(pages[i]);
669 pages[i] = NULL;
670 }
671 }
672 return;
673 }
674
675 mtx_lock(&pool->lock);
676 for (i = 0; i < npages; i++) {
677 if (pages[i]) {
678 TAILQ_INSERT_TAIL(&pool->list, pages[i], plinks.q);
679 pages[i] = NULL;
680 pool->npages++;
681 }
682 }
683 /* Check that we don't go over the pool limit */
684 npages = 0;
685 if (pool->npages > _manager->options.max_size) {
686 npages = pool->npages - _manager->options.max_size;
687 /* free at least NUM_PAGES_TO_ALLOC number of pages
688 * to reduce calls to set_memory_wb */
689 if (npages < NUM_PAGES_TO_ALLOC)
690 npages = NUM_PAGES_TO_ALLOC;
691 }
692 mtx_unlock(&pool->lock);
693 if (npages)
694 ttm_page_pool_free(pool, npages);
695}
696
697/*
698 * On success pages list will hold count number of correctly
699 * cached pages.
700 */
701static int ttm_get_pages(vm_page_t *pages, unsigned npages, int flags,
702 enum ttm_caching_state cstate)
703{
704 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
705 struct pglist plist;
706 vm_page_t p = NULL;
707 int gfp_flags, aflags;
708 unsigned count;
709 int r;
710
711 aflags = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
712 ((flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0 ? VM_ALLOC_ZERO : 0);
713
714 /* No pool for cached pages */
715 if (pool == NULL) {
716 for (r = 0; r < npages; ++r) {
717 p = vm_page_alloc_contig(NULL, 0, aflags, 1, 0,
718 (flags & TTM_PAGE_FLAG_DMA32) ? 0xffffffff :
719 VM_MAX_ADDRESS, PAGE_SIZE,
720 0, ttm_caching_state_to_vm(cstate));
721 if (!p) {
722 printf("[TTM] Unable to allocate page\n");
723 return -ENOMEM;
724 }
725 p->oflags &= ~VPO_UNMANAGED;
726 p->flags |= PG_FICTITIOUS;
727 pages[r] = p;
728 }
729 return 0;
730 }
731
732 /* combine zero flag to pool flags */
733 gfp_flags = flags | pool->ttm_page_alloc_flags;
734
735 /* First we take pages from the pool */
736 TAILQ_INIT(&plist);
737 npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
738 count = 0;
739 TAILQ_FOREACH(p, &plist, plinks.q) {
740 pages[count++] = p;
741 }
742
743 /* clear the pages coming from the pool if requested */
744 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
745 TAILQ_FOREACH(p, &plist, plinks.q) {
746 pmap_zero_page(p);
747 }
748 }
749
750 /* If pool didn't have enough pages allocate new one. */
751 if (npages > 0) {
752 /* ttm_alloc_new_pages doesn't reference pool so we can run
753 * multiple requests in parallel.
754 **/
755 TAILQ_INIT(&plist);
756 r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate,
757 npages);
758 TAILQ_FOREACH(p, &plist, plinks.q) {
759 pages[count++] = p;
760 }
761 if (r) {
762 /* If there is any pages in the list put them back to
763 * the pool. */
764 printf("[TTM] Failed to allocate extra pages for large request\n");
765 ttm_put_pages(pages, count, flags, cstate);
766 return r;
767 }
768 }
769
770 return 0;
771}
772
773static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
774 char *name)
775{
776 mtx_init(&pool->lock, "ttmpool", NULL, MTX_DEF);
777 pool->fill_lock = false;
778 TAILQ_INIT(&pool->list);
779 pool->npages = pool->nfrees = 0;
780 pool->ttm_page_alloc_flags = flags;
781 pool->name = name;
782}
783
784int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
785{
786
787 if (_manager != NULL)
788 printf("[TTM] manager != NULL\n");
789 printf("[TTM] Initializing pool allocator\n");
790
791 _manager = malloc(sizeof(*_manager), M_TTM_POOLMGR, M_WAITOK | M_ZERO);
792
793 ttm_page_pool_init_locked(&_manager->wc_pool, 0, "wc");
794 ttm_page_pool_init_locked(&_manager->uc_pool, 0, "uc");
795 ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
796 TTM_PAGE_FLAG_DMA32, "wc dma");
797 ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
798 TTM_PAGE_FLAG_DMA32, "uc dma");
799
800 _manager->options.max_size = max_pages;
801 _manager->options.small = SMALL_ALLOCATION;
802 _manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
803
804 refcount_init(&_manager->kobj_ref, 1);
805 ttm_pool_mm_shrink_init(_manager);
806
807 return 0;
808}
809
810void ttm_page_alloc_fini(void)
811{
812 int i;
813
814 printf("[TTM] Finalizing pool allocator\n");
815 ttm_pool_mm_shrink_fini(_manager);
816
817 for (i = 0; i < NUM_POOLS; ++i)
818 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES);
819
820 if (refcount_release(&_manager->kobj_ref))
821 ttm_pool_kobj_release(_manager);
822 _manager = NULL;
823}
824
825int ttm_pool_populate(struct ttm_tt *ttm)
826{
827 struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
828 unsigned i;
829 int ret;
830
831 if (ttm->state != tt_unpopulated)
832 return 0;
833
834 for (i = 0; i < ttm->num_pages; ++i) {
835 ret = ttm_get_pages(&ttm->pages[i], 1,
836 ttm->page_flags,
837 ttm->caching_state);
838 if (ret != 0) {
839 ttm_pool_unpopulate(ttm);
840 return -ENOMEM;
841 }
842
843 ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
844 false, false);
845 if (unlikely(ret != 0)) {
846 ttm_pool_unpopulate(ttm);
847 return -ENOMEM;
848 }
849 }
850
851 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
852 ret = ttm_tt_swapin(ttm);
853 if (unlikely(ret != 0)) {
854 ttm_pool_unpopulate(ttm);
855 return ret;
856 }
857 }
858
859 ttm->state = tt_unbound;
860 return 0;
861}
862
863void ttm_pool_unpopulate(struct ttm_tt *ttm)
864{
865 unsigned i;
866
867 for (i = 0; i < ttm->num_pages; ++i) {
868 if (ttm->pages[i]) {
869 ttm_mem_global_free_page(ttm->glob->mem_glob,
870 ttm->pages[i]);
871 ttm_put_pages(&ttm->pages[i], 1,
872 ttm->page_flags,
873 ttm->caching_state);
874 }
875 }
876 ttm->state = tt_unpopulated;
877}
878
879#if 0
880/* XXXKIB sysctl */
881int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
882{
883 struct ttm_page_pool *p;
884 unsigned i;
885 char *h[] = {"pool", "refills", "pages freed", "size"};
886 if (!_manager) {
887 seq_printf(m, "No pool allocator running.\n");
888 return 0;
889 }
890 seq_printf(m, "%6s %12s %13s %8s\n",
891 h[0], h[1], h[2], h[3]);
892 for (i = 0; i < NUM_POOLS; ++i) {
893 p = &_manager->pools[i];
894
895 seq_printf(m, "%6s %12ld %13ld %8d\n",
896 p->name, p->nrefills,
897 p->nfrees, p->npages);
898 }
899 return 0;
900}
901#endif
| 247 return 0;
248}
249
250/**
251 * Select the right pool or requested caching state and ttm flags. */
252static struct ttm_page_pool *ttm_get_pool(int flags,
253 enum ttm_caching_state cstate)
254{
255 int pool_index;
256
257 if (cstate == tt_cached)
258 return NULL;
259
260 if (cstate == tt_wc)
261 pool_index = 0x0;
262 else
263 pool_index = 0x1;
264
265 if (flags & TTM_PAGE_FLAG_DMA32)
266 pool_index |= 0x2;
267
268 return &_manager->pools[pool_index];
269}
270
271/* set memory back to wb and free the pages. */
272static void ttm_pages_put(vm_page_t *pages, unsigned npages)
273{
274 unsigned i;
275
276 /* Our VM handles vm memattr automatically on the page free. */
277 if (set_pages_array_wb(pages, npages))
278 printf("[TTM] Failed to set %d pages to wb!\n", npages);
279 for (i = 0; i < npages; ++i)
280 ttm_vm_page_free(pages[i]);
281}
282
283static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
284 unsigned freed_pages)
285{
286 pool->npages -= freed_pages;
287 pool->nfrees += freed_pages;
288}
289
290/**
291 * Free pages from pool.
292 *
293 * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
294 * number of pages in one go.
295 *
296 * @pool: to free the pages from
297 * @free_all: If set to true will free all pages in pool
298 **/
299static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free)
300{
301 vm_page_t p, p1;
302 vm_page_t *pages_to_free;
303 unsigned freed_pages = 0,
304 npages_to_free = nr_free;
305 unsigned i;
306
307 if (NUM_PAGES_TO_ALLOC < nr_free)
308 npages_to_free = NUM_PAGES_TO_ALLOC;
309
310 pages_to_free = malloc(npages_to_free * sizeof(vm_page_t),
311 M_TEMP, M_WAITOK | M_ZERO);
312
313restart:
314 mtx_lock(&pool->lock);
315
316 TAILQ_FOREACH_REVERSE_SAFE(p, &pool->list, pglist, plinks.q, p1) {
317 if (freed_pages >= npages_to_free)
318 break;
319
320 pages_to_free[freed_pages++] = p;
321 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */
322 if (freed_pages >= NUM_PAGES_TO_ALLOC) {
323 /* remove range of pages from the pool */
324 for (i = 0; i < freed_pages; i++)
325 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q);
326
327 ttm_pool_update_free_locked(pool, freed_pages);
328 /**
329 * Because changing page caching is costly
330 * we unlock the pool to prevent stalling.
331 */
332 mtx_unlock(&pool->lock);
333
334 ttm_pages_put(pages_to_free, freed_pages);
335 if (likely(nr_free != FREE_ALL_PAGES))
336 nr_free -= freed_pages;
337
338 if (NUM_PAGES_TO_ALLOC >= nr_free)
339 npages_to_free = nr_free;
340 else
341 npages_to_free = NUM_PAGES_TO_ALLOC;
342
343 freed_pages = 0;
344
345 /* free all so restart the processing */
346 if (nr_free)
347 goto restart;
348
349 /* Not allowed to fall through or break because
350 * following context is inside spinlock while we are
351 * outside here.
352 */
353 goto out;
354
355 }
356 }
357
358 /* remove range of pages from the pool */
359 if (freed_pages) {
360 for (i = 0; i < freed_pages; i++)
361 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q);
362
363 ttm_pool_update_free_locked(pool, freed_pages);
364 nr_free -= freed_pages;
365 }
366
367 mtx_unlock(&pool->lock);
368
369 if (freed_pages)
370 ttm_pages_put(pages_to_free, freed_pages);
371out:
372 free(pages_to_free, M_TEMP);
373 return nr_free;
374}
375
376/* Get good estimation how many pages are free in pools */
377static int ttm_pool_get_num_unused_pages(void)
378{
379 unsigned i;
380 int total = 0;
381 for (i = 0; i < NUM_POOLS; ++i)
382 total += _manager->pools[i].npages;
383
384 return total;
385}
386
387/**
388 * Callback for mm to request pool to reduce number of page held.
389 */
390static int ttm_pool_mm_shrink(void *arg)
391{
392 static unsigned int start_pool = 0;
393 unsigned i;
394 unsigned pool_offset = atomic_fetchadd_int(&start_pool, 1);
395 struct ttm_page_pool *pool;
396 int shrink_pages = 100; /* XXXKIB */
397
398 pool_offset = pool_offset % NUM_POOLS;
399 /* select start pool in round robin fashion */
400 for (i = 0; i < NUM_POOLS; ++i) {
401 unsigned nr_free = shrink_pages;
402 if (shrink_pages == 0)
403 break;
404 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
405 shrink_pages = ttm_page_pool_free(pool, nr_free);
406 }
407 /* return estimated number of unused pages in pool */
408 return ttm_pool_get_num_unused_pages();
409}
410
411static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
412{
413
414 manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem,
415 ttm_pool_mm_shrink, manager, EVENTHANDLER_PRI_ANY);
416}
417
418static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
419{
420
421 EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler);
422}
423
424static int ttm_set_pages_caching(vm_page_t *pages,
425 enum ttm_caching_state cstate, unsigned cpages)
426{
427 int r = 0;
428 /* Set page caching */
429 switch (cstate) {
430 case tt_uncached:
431 r = set_pages_array_uc(pages, cpages);
432 if (r)
433 printf("[TTM] Failed to set %d pages to uc!\n", cpages);
434 break;
435 case tt_wc:
436 r = set_pages_array_wc(pages, cpages);
437 if (r)
438 printf("[TTM] Failed to set %d pages to wc!\n", cpages);
439 break;
440 default:
441 break;
442 }
443 return r;
444}
445
446/**
447 * Free pages the pages that failed to change the caching state. If there is
448 * any pages that have changed their caching state already put them to the
449 * pool.
450 */
451static void ttm_handle_caching_state_failure(struct pglist *pages,
452 int ttm_flags, enum ttm_caching_state cstate,
453 vm_page_t *failed_pages, unsigned cpages)
454{
455 unsigned i;
456 /* Failed pages have to be freed */
457 for (i = 0; i < cpages; ++i) {
458 TAILQ_REMOVE(pages, failed_pages[i], plinks.q);
459 ttm_vm_page_free(failed_pages[i]);
460 }
461}
462
463/**
464 * Allocate new pages with correct caching.
465 *
466 * This function is reentrant if caller updates count depending on number of
467 * pages returned in pages array.
468 */
469static int ttm_alloc_new_pages(struct pglist *pages, int ttm_alloc_flags,
470 int ttm_flags, enum ttm_caching_state cstate, unsigned count)
471{
472 vm_page_t *caching_array;
473 vm_page_t p;
474 int r = 0;
475 unsigned i, cpages, aflags;
476 unsigned max_cpages = min(count,
477 (unsigned)(PAGE_SIZE/sizeof(vm_page_t)));
478
479 aflags = VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ |
480 ((ttm_alloc_flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0 ?
481 VM_ALLOC_ZERO : 0);
482
483 /* allocate array for page caching change */
484 caching_array = malloc(max_cpages * sizeof(vm_page_t), M_TEMP,
485 M_WAITOK | M_ZERO);
486
487 for (i = 0, cpages = 0; i < count; ++i) {
488 p = vm_page_alloc_contig(NULL, 0, aflags, 1, 0,
489 (ttm_alloc_flags & TTM_PAGE_FLAG_DMA32) ? 0xffffffff :
490 VM_MAX_ADDRESS, PAGE_SIZE, 0,
491 ttm_caching_state_to_vm(cstate));
492 if (!p) {
493 printf("[TTM] Unable to get page %u\n", i);
494
495 /* store already allocated pages in the pool after
496 * setting the caching state */
497 if (cpages) {
498 r = ttm_set_pages_caching(caching_array,
499 cstate, cpages);
500 if (r)
501 ttm_handle_caching_state_failure(pages,
502 ttm_flags, cstate,
503 caching_array, cpages);
504 }
505 r = -ENOMEM;
506 goto out;
507 }
508 p->oflags &= ~VPO_UNMANAGED;
509 p->flags |= PG_FICTITIOUS;
510
511#ifdef CONFIG_HIGHMEM /* KIB: nop */
512 /* gfp flags of highmem page should never be dma32 so we
513 * we should be fine in such case
514 */
515 if (!PageHighMem(p))
516#endif
517 {
518 caching_array[cpages++] = p;
519 if (cpages == max_cpages) {
520
521 r = ttm_set_pages_caching(caching_array,
522 cstate, cpages);
523 if (r) {
524 ttm_handle_caching_state_failure(pages,
525 ttm_flags, cstate,
526 caching_array, cpages);
527 goto out;
528 }
529 cpages = 0;
530 }
531 }
532
533 TAILQ_INSERT_HEAD(pages, p, plinks.q);
534 }
535
536 if (cpages) {
537 r = ttm_set_pages_caching(caching_array, cstate, cpages);
538 if (r)
539 ttm_handle_caching_state_failure(pages,
540 ttm_flags, cstate,
541 caching_array, cpages);
542 }
543out:
544 free(caching_array, M_TEMP);
545
546 return r;
547}
548
549/**
550 * Fill the given pool if there aren't enough pages and the requested number of
551 * pages is small.
552 */
553static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
554 int ttm_flags, enum ttm_caching_state cstate, unsigned count)
555{
556 vm_page_t p;
557 int r;
558 unsigned cpages = 0;
559 /**
560 * Only allow one pool fill operation at a time.
561 * If pool doesn't have enough pages for the allocation new pages are
562 * allocated from outside of pool.
563 */
564 if (pool->fill_lock)
565 return;
566
567 pool->fill_lock = true;
568
569 /* If allocation request is small and there are not enough
570 * pages in a pool we fill the pool up first. */
571 if (count < _manager->options.small
572 && count > pool->npages) {
573 struct pglist new_pages;
574 unsigned alloc_size = _manager->options.alloc_size;
575
576 /**
577 * Can't change page caching if in irqsave context. We have to
578 * drop the pool->lock.
579 */
580 mtx_unlock(&pool->lock);
581
582 TAILQ_INIT(&new_pages);
583 r = ttm_alloc_new_pages(&new_pages, pool->ttm_page_alloc_flags,
584 ttm_flags, cstate, alloc_size);
585 mtx_lock(&pool->lock);
586
587 if (!r) {
588 TAILQ_CONCAT(&pool->list, &new_pages, plinks.q);
589 ++pool->nrefills;
590 pool->npages += alloc_size;
591 } else {
592 printf("[TTM] Failed to fill pool (%p)\n", pool);
593 /* If we have any pages left put them to the pool. */
594 TAILQ_FOREACH(p, &pool->list, plinks.q) {
595 ++cpages;
596 }
597 TAILQ_CONCAT(&pool->list, &new_pages, plinks.q);
598 pool->npages += cpages;
599 }
600
601 }
602 pool->fill_lock = false;
603}
604
605/**
606 * Cut 'count' number of pages from the pool and put them on the return list.
607 *
608 * @return count of pages still required to fulfill the request.
609 */
610static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
611 struct pglist *pages,
612 int ttm_flags,
613 enum ttm_caching_state cstate,
614 unsigned count)
615{
616 vm_page_t p;
617 unsigned i;
618
619 mtx_lock(&pool->lock);
620 ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count);
621
622 if (count >= pool->npages) {
623 /* take all pages from the pool */
624 TAILQ_CONCAT(pages, &pool->list, plinks.q);
625 count -= pool->npages;
626 pool->npages = 0;
627 goto out;
628 }
629 for (i = 0; i < count; i++) {
630 p = TAILQ_FIRST(&pool->list);
631 TAILQ_REMOVE(&pool->list, p, plinks.q);
632 TAILQ_INSERT_TAIL(pages, p, plinks.q);
633 }
634 pool->npages -= count;
635 count = 0;
636out:
637 mtx_unlock(&pool->lock);
638 return count;
639}
640
641/* Put all pages in pages list to correct pool to wait for reuse */
642static void ttm_put_pages(vm_page_t *pages, unsigned npages, int flags,
643 enum ttm_caching_state cstate)
644{
645 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
646 unsigned i;
647
648 if (pool == NULL) {
649 /* No pool for this memory type so free the pages */
650 for (i = 0; i < npages; i++) {
651 if (pages[i]) {
652 ttm_vm_page_free(pages[i]);
653 pages[i] = NULL;
654 }
655 }
656 return;
657 }
658
659 mtx_lock(&pool->lock);
660 for (i = 0; i < npages; i++) {
661 if (pages[i]) {
662 TAILQ_INSERT_TAIL(&pool->list, pages[i], plinks.q);
663 pages[i] = NULL;
664 pool->npages++;
665 }
666 }
667 /* Check that we don't go over the pool limit */
668 npages = 0;
669 if (pool->npages > _manager->options.max_size) {
670 npages = pool->npages - _manager->options.max_size;
671 /* free at least NUM_PAGES_TO_ALLOC number of pages
672 * to reduce calls to set_memory_wb */
673 if (npages < NUM_PAGES_TO_ALLOC)
674 npages = NUM_PAGES_TO_ALLOC;
675 }
676 mtx_unlock(&pool->lock);
677 if (npages)
678 ttm_page_pool_free(pool, npages);
679}
680
681/*
682 * On success pages list will hold count number of correctly
683 * cached pages.
684 */
685static int ttm_get_pages(vm_page_t *pages, unsigned npages, int flags,
686 enum ttm_caching_state cstate)
687{
688 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
689 struct pglist plist;
690 vm_page_t p = NULL;
691 int gfp_flags, aflags;
692 unsigned count;
693 int r;
694
695 aflags = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
696 ((flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0 ? VM_ALLOC_ZERO : 0);
697
698 /* No pool for cached pages */
699 if (pool == NULL) {
700 for (r = 0; r < npages; ++r) {
701 p = vm_page_alloc_contig(NULL, 0, aflags, 1, 0,
702 (flags & TTM_PAGE_FLAG_DMA32) ? 0xffffffff :
703 VM_MAX_ADDRESS, PAGE_SIZE,
704 0, ttm_caching_state_to_vm(cstate));
705 if (!p) {
706 printf("[TTM] Unable to allocate page\n");
707 return -ENOMEM;
708 }
709 p->oflags &= ~VPO_UNMANAGED;
710 p->flags |= PG_FICTITIOUS;
711 pages[r] = p;
712 }
713 return 0;
714 }
715
716 /* combine zero flag to pool flags */
717 gfp_flags = flags | pool->ttm_page_alloc_flags;
718
719 /* First we take pages from the pool */
720 TAILQ_INIT(&plist);
721 npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
722 count = 0;
723 TAILQ_FOREACH(p, &plist, plinks.q) {
724 pages[count++] = p;
725 }
726
727 /* clear the pages coming from the pool if requested */
728 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
729 TAILQ_FOREACH(p, &plist, plinks.q) {
730 pmap_zero_page(p);
731 }
732 }
733
734 /* If pool didn't have enough pages allocate new one. */
735 if (npages > 0) {
736 /* ttm_alloc_new_pages doesn't reference pool so we can run
737 * multiple requests in parallel.
738 **/
739 TAILQ_INIT(&plist);
740 r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate,
741 npages);
742 TAILQ_FOREACH(p, &plist, plinks.q) {
743 pages[count++] = p;
744 }
745 if (r) {
746 /* If there is any pages in the list put them back to
747 * the pool. */
748 printf("[TTM] Failed to allocate extra pages for large request\n");
749 ttm_put_pages(pages, count, flags, cstate);
750 return r;
751 }
752 }
753
754 return 0;
755}
756
757static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
758 char *name)
759{
760 mtx_init(&pool->lock, "ttmpool", NULL, MTX_DEF);
761 pool->fill_lock = false;
762 TAILQ_INIT(&pool->list);
763 pool->npages = pool->nfrees = 0;
764 pool->ttm_page_alloc_flags = flags;
765 pool->name = name;
766}
767
768int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
769{
770
771 if (_manager != NULL)
772 printf("[TTM] manager != NULL\n");
773 printf("[TTM] Initializing pool allocator\n");
774
775 _manager = malloc(sizeof(*_manager), M_TTM_POOLMGR, M_WAITOK | M_ZERO);
776
777 ttm_page_pool_init_locked(&_manager->wc_pool, 0, "wc");
778 ttm_page_pool_init_locked(&_manager->uc_pool, 0, "uc");
779 ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
780 TTM_PAGE_FLAG_DMA32, "wc dma");
781 ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
782 TTM_PAGE_FLAG_DMA32, "uc dma");
783
784 _manager->options.max_size = max_pages;
785 _manager->options.small = SMALL_ALLOCATION;
786 _manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
787
788 refcount_init(&_manager->kobj_ref, 1);
789 ttm_pool_mm_shrink_init(_manager);
790
791 return 0;
792}
793
794void ttm_page_alloc_fini(void)
795{
796 int i;
797
798 printf("[TTM] Finalizing pool allocator\n");
799 ttm_pool_mm_shrink_fini(_manager);
800
801 for (i = 0; i < NUM_POOLS; ++i)
802 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES);
803
804 if (refcount_release(&_manager->kobj_ref))
805 ttm_pool_kobj_release(_manager);
806 _manager = NULL;
807}
808
809int ttm_pool_populate(struct ttm_tt *ttm)
810{
811 struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
812 unsigned i;
813 int ret;
814
815 if (ttm->state != tt_unpopulated)
816 return 0;
817
818 for (i = 0; i < ttm->num_pages; ++i) {
819 ret = ttm_get_pages(&ttm->pages[i], 1,
820 ttm->page_flags,
821 ttm->caching_state);
822 if (ret != 0) {
823 ttm_pool_unpopulate(ttm);
824 return -ENOMEM;
825 }
826
827 ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
828 false, false);
829 if (unlikely(ret != 0)) {
830 ttm_pool_unpopulate(ttm);
831 return -ENOMEM;
832 }
833 }
834
835 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
836 ret = ttm_tt_swapin(ttm);
837 if (unlikely(ret != 0)) {
838 ttm_pool_unpopulate(ttm);
839 return ret;
840 }
841 }
842
843 ttm->state = tt_unbound;
844 return 0;
845}
846
847void ttm_pool_unpopulate(struct ttm_tt *ttm)
848{
849 unsigned i;
850
851 for (i = 0; i < ttm->num_pages; ++i) {
852 if (ttm->pages[i]) {
853 ttm_mem_global_free_page(ttm->glob->mem_glob,
854 ttm->pages[i]);
855 ttm_put_pages(&ttm->pages[i], 1,
856 ttm->page_flags,
857 ttm->caching_state);
858 }
859 }
860 ttm->state = tt_unpopulated;
861}
862
863#if 0
864/* XXXKIB sysctl */
865int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
866{
867 struct ttm_page_pool *p;
868 unsigned i;
869 char *h[] = {"pool", "refills", "pages freed", "size"};
870 if (!_manager) {
871 seq_printf(m, "No pool allocator running.\n");
872 return 0;
873 }
874 seq_printf(m, "%6s %12s %13s %8s\n",
875 h[0], h[1], h[2], h[3]);
876 for (i = 0; i < NUM_POOLS; ++i) {
877 p = &_manager->pools[i];
878
879 seq_printf(m, "%6s %12ld %13ld %8d\n",
880 p->name, p->nrefills,
881 p->nfrees, p->npages);
882 }
883 return 0;
884}
885#endif
|