drivers/iommu/iova.c

366  * alloc_iova_fast - allocates an iova from rcache
370  * @flush_rcache: - set to flush rcache on regular allocation failure
371  * This function tries to satisfy an iova allocation from the rcache,
373  * fails too and the flush_rcache flag is set then the rcache will be flushed.
403 		/* Try replenishing IOVAs by flushing rcache. */
416  * free_iova_fast - free iova pfn range into rcache
420  * This functions frees an iova range by trying to put it into the rcache,
672 static struct iova_magazine *iova_depot_pop(struct iova_rcache *rcache)
674 	struct iova_magazine *mag = rcache->depot;
676 	rcache->depot = mag->next;
678 	rcache->depot_size--;
682 static void iova_depot_push(struct iova_rcache *rcache, struct iova_magazine *mag)
684 	mag->next = rcache->depot;
685 	rcache->depot = mag;
686 	rcache->depot_size++;
691 	struct iova_rcache *rcache = container_of(work, typeof(*rcache), work.work);
695 	spin_lock_irqsave(&rcache->lock, flags);
696 	if (rcache->depot_size > num_online_cpus())
697 		mag = iova_depot_pop(rcache);
698 	spin_unlock_irqrestore(&rcache->lock, flags);
701 		iova_magazine_free_pfns(mag, rcache->iovad);
703 		schedule_delayed_work(&rcache->work, IOVA_DEPOT_DELAY);
720 		struct iova_rcache *rcache;
722 		rcache = &iovad->rcaches[i];
723 		spin_lock_init(&rcache->lock);
724 		rcache->iovad = iovad;
725 		INIT_DELAYED_WORK(&rcache->work, iova_depot_work_func);
726 		rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache),
728 		if (!rcache->cpu_rcaches) {
733 			cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
758  * Try inserting IOVA range starting with 'iova_pfn' into 'rcache', and
759  * return true on success.  Can fail if rcache is full and we can't free
764 				 struct iova_rcache *rcache,
771 	cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
783 			spin_lock(&rcache->lock);
784 			iova_depot_push(rcache, cpu_rcache->loaded);
785 			spin_unlock(&rcache->lock);
786 			schedule_delayed_work(&rcache->work, IOVA_DEPOT_DELAY);
813  * Caller wants to allocate a new IOVA range from 'rcache'.  If we can
815  * it from the 'rcache'.
817 static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
825 	cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
834 		spin_lock(&rcache->lock);
835 		if (rcache->depot) {
837 			cpu_rcache->loaded = iova_depot_pop(rcache);
840 		spin_unlock(&rcache->lock);
852  * Try to satisfy IOVA allocation range from rcache.  Fail if requested
869  * free rcache data structures.
873 	struct iova_rcache *rcache;
878 		rcache = &iovad->rcaches[i];
879 		if (!rcache->cpu_rcaches)
882 			cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
886 		free_percpu(rcache->cpu_rcaches);
887 		cancel_delayed_work_sync(&rcache->work);
888 		while (rcache->depot)
889 			iova_magazine_free(iova_depot_pop(rcache));
902 	struct iova_rcache *rcache;
907 		rcache = &iovad->rcaches[i];
908 		cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
921 	struct iova_rcache *rcache;
925 		rcache = &iovad->rcaches[i];
926 		spin_lock_irqsave(&rcache->lock, flags);
927 		while (rcache->depot) {
928 			struct iova_magazine *mag = iova_depot_pop(rcache);
933 		spin_unlock_irqrestore(&rcache->lock, flags);