x86/x86/busdma_bounce.c

121 	bus_dma_tag_t	       dmat;
133 static int alloc_bounce_zone(bus_dma_tag_t dmat);
134 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
135 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
137 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
140 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
141 static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
144 static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
147 static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
158     void *lockfuncarg, bus_dma_tag_t *dmat)
163 	*dmat = NULL;
211 		*dmat = newtag;
219 bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat)
225 	dmat_copy = dmat;
227 	if (dmat != NULL) {
228 		if (dmat->map_count != 0) {
232 		while (dmat != NULL) {
233 			parent = (bus_dma_tag_t)dmat->common.parent;
234 			atomic_subtract_int(&dmat->common.ref_count, 1);
235 			if (dmat->common.ref_count == 0) {
236 				if (dmat->segments != NULL)
237 					free(dmat->segments, M_DEVBUF);
238 				free(dmat, M_DEVBUF);
244 				dmat = parent;
246 				dmat = NULL;
259 bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
266 	if (dmat->segments == NULL) {
267 		dmat->segments = (bus_dma_segment_t *)malloc(
268 		    sizeof(bus_dma_segment_t) * dmat->common.nsegments,
270 		if (dmat->segments == NULL) {
272 			    __func__, dmat, ENOMEM);
282 	if (dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) {
284 		if (dmat->bounce_zone == NULL) {
285 			if ((error = alloc_bounce_zone(dmat)) != 0)
288 		bz = dmat->bounce_zone;
294 			    __func__, dmat, ENOMEM);
305 		if (dmat->common.alignment > 1)
309 			    atop(dmat->common.lowaddr));
310 		if ((dmat->bounce_flags & BUS_DMA_MIN_ALLOC_COMP) == 0 ||
312 			pages = MAX(atop(dmat->common.maxsize), 1);
315 			if (alloc_bounce_pages(dmat, pages) < pages)
317 			if ((dmat->bounce_flags & BUS_DMA_MIN_ALLOC_COMP)
320 					dmat->bounce_flags |=
331 		dmat->map_count++;
333 	    __func__, dmat, dmat->common.flags, error);
342 bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
348 			    __func__, dmat, EBUSY);
351 		if (dmat->bounce_zone)
352 			dmat->bounce_zone->map_count--;
355 	dmat->map_count--;
356 	CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
367 bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
381 	if (dmat->segments == NULL) {
382 		dmat->segments = (bus_dma_segment_t *)malloc(
383 		    sizeof(bus_dma_segment_t) * dmat->common.nsegments,
385 		if (dmat->segments == NULL) {
387 			    __func__, dmat, dmat->common.flags, ENOMEM);
412 	 * NOTE: The (dmat->common.alignment <= dmat->maxsize) check
419 	if ((dmat->common.maxsize <= PAGE_SIZE) &&
420 	   (dmat->common.alignment <= dmat->common.maxsize) &&
421 	    dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
423 		*vaddr = malloc(dmat->common.maxsize, M_DEVBUF, mflags);
424 	} else if (dmat->common.nsegments >=
425 	    howmany(dmat->common.maxsize, MIN(dmat->common.maxsegsz, PAGE_SIZE)) &&
426 	    dmat->common.alignment <= PAGE_SIZE &&
427 	    (dmat->common.boundary % PAGE_SIZE) == 0) {
430 		    dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr,
432 		dmat->bounce_flags |= BUS_DMA_KMEM_ALLOC;
435 		    dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr,
436 		    dmat->common.alignment != 0 ? dmat->common.alignment : 1ul,
437 		    dmat->common.boundary, attr);
438 		dmat->bounce_flags |= BUS_DMA_KMEM_ALLOC;
442 		    __func__, dmat, dmat->common.flags, ENOMEM);
444 	} else if (vtophys(*vaddr) & (dmat->common.alignment - 1)) {
448 	    __func__, dmat, dmat->common.flags, 0);
457 bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
466 	if ((dmat->bounce_flags & BUS_DMA_KMEM_ALLOC) == 0)
470 		    dmat->common.maxsize);
471 	CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat,
472 	    dmat->bounce_flags);
476 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
489 			sgsize = MIN(buflen, dmat->common.maxsegsz);
490 			if (bus_dma_run_filter(&dmat->common, curaddr)) {
503 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
513 		    "alignment= %d", dmat->common.lowaddr,
515 		    dmat->common.boundary, dmat->common.alignment);
531 			if (bus_dma_run_filter(&dmat->common, paddr) != 0) {
533 				    dmat->common.alignment);
543 _bus_dmamap_count_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma,
552 		    "alignment= %d", dmat->common.lowaddr,
554 		    dmat->common.boundary, dmat->common.alignment);
566 			max_sgsize = MIN(buflen, dmat->common.maxsegsz);
568 			if (bus_dma_run_filter(&dmat->common, paddr) != 0) {
570 				    dmat->common.alignment);
572 				KASSERT((sg_len & (dmat->common.alignment - 1))
588 _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
594 		if (reserve_bounce_pages(dmat, map, 0) != 0) {
599 		if (reserve_bounce_pages(dmat, map, 1) != 0) {
615 _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
624 	bmask = ~(dmat->common.boundary - 1);
625 	if (dmat->common.boundary > 0) {
626 		baddr = (curaddr + dmat->common.boundary) & bmask;
642 		    (segs[seg].ds_len + sgsize) <= dmat->common.maxsegsz &&
643 		    (dmat->common.boundary == 0 ||
647 			if (++seg >= dmat->common.nsegments)
662 bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
674 		segs = dmat->segments;
676 	if ((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) {
677 		_bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
679 			error = _bus_dmamap_reserve_pages(dmat, map, flags);
687 		sgsize = MIN(buflen, dmat->common.maxsegsz);
688 		if (((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) &&
690 		    bus_dma_run_filter(&dmat->common, curaddr)) {
692 			curaddr = add_bounce_page(dmat, map, 0, curaddr, 0,
695 		sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
714 bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
727 		segs = dmat->segments;
729 	if ((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) {
730 		_bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
732 			error = _bus_dmamap_reserve_pages(dmat, map, flags);
754 		max_sgsize = MIN(buflen, dmat->common.maxsegsz);
756 		if (((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) &&
758 		    bus_dma_run_filter(&dmat->common, curaddr)) {
759 			sgsize = roundup2(sgsize, dmat->common.alignment);
761 			curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 0,
766 		sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
781 bounce_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map,
789 	if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) {
795 		error = bus_dmamap_load_ma_triv(dmat, map, ma, buflen, ma_offs,
804 		segs = dmat->segments;
806 	if ((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) {
807 		_bus_dmamap_count_ma(dmat, map, ma, ma_offs, buflen, flags);
809 			error = _bus_dmamap_reserve_pages(dmat, map, flags);
821 		max_sgsize = MIN(buflen, dmat->common.maxsegsz);
823 		if (((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) &&
825 		    bus_dma_run_filter(&dmat->common, paddr)) {
826 			sgsize = roundup2(sgsize, dmat->common.alignment);
828 			KASSERT((sgsize & (dmat->common.alignment - 1)) == 0,
839 			paddr = add_bounce_page(dmat, map, 0, paddr,
844 		sgsize = _bus_dmamap_addseg(dmat, map, paddr, sgsize, segs,
863 bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
870 	map->dmat = dmat;
876 bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
881 		segs = dmat->segments;
889 bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
898 		free_bounce_page(dmat, bpage);
903 bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
918 	    "performing bounce", __func__, dmat, dmat->common.flags, op);
958 		dmat->bounce_zone->total_bounced++;
999 		dmat->bounce_zone->total_bounced++;
1030 alloc_bounce_zone(bus_dma_tag_t dmat)
1036 		if ((dmat->common.alignment <= bz->alignment) &&
1037 		    (dmat->common.lowaddr >= bz->lowaddr)) {
1038 			dmat->bounce_zone = bz;
1051 	bz->lowaddr = dmat->common.lowaddr;
1052 	bz->alignment = MAX(dmat->common.alignment, PAGE_SIZE);
1058 	dmat->bounce_zone = bz;
1104 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1109 	bz = dmat->bounce_zone;
1142 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1148 	bz = dmat->bounce_zone;
1161 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1167 	KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1171 	bz = dmat->bounce_zone;
1190 	if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) {
1208 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1213 	bz = dmat->bounce_zone;
1216 	if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) {
1231 		if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1246 	bus_dma_tag_t dmat;
1253 		dmat = map->dmat;
1254 		(dmat->common.lockfunc)(dmat->common.lockfuncarg, BUS_DMA_LOCK);
1255 		bus_dmamap_load_mem(map->dmat, map, &map->mem,
1257 		(dmat->common.lockfunc)(dmat->common.lockfuncarg,