1/* 2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. 3 * Portions based on net/core/datagram.c and copyrighted by their authors. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License as published by the Free 7 * Software Foundation; either version 2 of the License, or (at your option) 8 * any later version. 9 * 10 * This program is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along with 16 * this program; if not, write to the Free Software Foundation, Inc., 59 17 * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * 19 * The full GNU General Public License is included in this distribution in the 20 * file called COPYING. 21 */ 22 23/* 24 * This code allows the net stack to make use of a DMA engine for 25 * skb to iovec copies. 26 */ 27 28#include <linux/dmaengine.h> 29#include <linux/pagemap.h> 30#include <linux/slab.h> 31#include <net/tcp.h> /* for memcpy_toiovec */ 32#include <asm/io.h> 33#include <asm/uaccess.h> 34 35static int num_pages_spanned(struct iovec *iov) 36{ 37 return 38 ((PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) - 39 ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT); 40} 41 42/* 43 * Pin down all the iovec pages needed for len bytes. 44 * Return a struct dma_pinned_list to keep track of pages pinned down. 45 * 46 * We are allocating a single chunk of memory, and then carving it up into 47 * 3 sections, the latter 2 whose size depends on the number of iovecs and the 48 * total number of pages, respectively. 49 */ 50struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len) 51{ 52 struct dma_pinned_list *local_list; 53 struct page **pages; 54 int i; 55 int ret; 56 int nr_iovecs = 0; 57 int iovec_len_used = 0; 58 int iovec_pages_used = 0; 59 60 /* don't pin down non-user-based iovecs */ 61 if (segment_eq(get_fs(), KERNEL_DS)) 62 return NULL; 63 64 /* determine how many iovecs/pages there are, up front */ 65 do { 66 iovec_len_used += iov[nr_iovecs].iov_len; 67 iovec_pages_used += num_pages_spanned(&iov[nr_iovecs]); 68 nr_iovecs++; 69 } while (iovec_len_used < len); 70 71 /* single kmalloc for pinned list, page_list[], and the page arrays */ 72 local_list = kmalloc(sizeof(*local_list) 73 + (nr_iovecs * sizeof (struct dma_page_list)) 74 + (iovec_pages_used * sizeof (struct page*)), GFP_KERNEL); 75 if (!local_list) 76 goto out; 77 78 /* list of pages starts right after the page list array */ 79 pages = (struct page **) &local_list->page_list[nr_iovecs]; 80 81 local_list->nr_iovecs = 0; 82 83 for (i = 0; i < nr_iovecs; i++) { 84 struct dma_page_list *page_list = &local_list->page_list[i]; 85 86 len -= iov[i].iov_len; 87 88 if (!access_ok(VERIFY_WRITE, iov[i].iov_base, iov[i].iov_len)) 89 goto unpin; 90 91 page_list->nr_pages = num_pages_spanned(&iov[i]); 92 page_list->base_address = iov[i].iov_base; 93 94 page_list->pages = pages; 95 pages += page_list->nr_pages; 96 97 /* pin pages down */ 98 down_read(¤t->mm->mmap_sem); 99 ret = get_user_pages( 100 current, 101 current->mm, 102 (unsigned long) iov[i].iov_base, 103 page_list->nr_pages, 104 1, /* write */ 105 0, /* force */ 106 page_list->pages, 107 NULL); 108 up_read(¤t->mm->mmap_sem); 109 110 if (ret != page_list->nr_pages) 111 goto unpin; 112 113 local_list->nr_iovecs = i + 1; 114 } 115 116 return local_list; 117 118unpin: 119 dma_unpin_iovec_pages(local_list); 120out: 121 return NULL; 122} 123 124void dma_unpin_iovec_pages(struct dma_pinned_list *pinned_list) 125{ 126 int i, j; 127 128 if (!pinned_list) 129 return; 130 131 for (i = 0; i < pinned_list->nr_iovecs; i++) { 132 struct dma_page_list *page_list = &pinned_list->page_list[i]; 133 for (j = 0; j < page_list->nr_pages; j++) { 134 set_page_dirty_lock(page_list->pages[j]); 135 page_cache_release(page_list->pages[j]); 136 } 137 } 138 139 kfree(pinned_list); 140} 141 142 143/* 144 * We have already pinned down the pages we will be using in the iovecs. 145 * Each entry in iov array has corresponding entry in pinned_list->page_list. 146 * Using array indexing to keep iov[] and page_list[] in sync. 147 * Initial elements in iov array's iov->iov_len will be 0 if already copied into 148 * by another call. 149 * iov array length remaining guaranteed to be bigger than len. 150 */ 151dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov, 152 struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len) 153{ 154 int iov_byte_offset; 155 int copy; 156 dma_cookie_t dma_cookie = 0; 157 int iovec_idx; 158 int page_idx; 159 160 if (!chan) 161 return memcpy_toiovec(iov, kdata, len); 162 163 iovec_idx = 0; 164 while (iovec_idx < pinned_list->nr_iovecs) { 165 struct dma_page_list *page_list; 166 167 /* skip already used-up iovecs */ 168 while (!iov[iovec_idx].iov_len) 169 iovec_idx++; 170 171 page_list = &pinned_list->page_list[iovec_idx]; 172 173 iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); 174 page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) 175 - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; 176 177 /* break up copies to not cross page boundary */ 178 while (iov[iovec_idx].iov_len) { 179 copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); 180 copy = min_t(int, copy, iov[iovec_idx].iov_len); 181 182 dma_cookie = dma_async_memcpy_buf_to_pg(chan, 183 page_list->pages[page_idx], 184 iov_byte_offset, 185 kdata, 186 copy); 187 /* poll for a descriptor slot */ 188 if (unlikely(dma_cookie < 0)) { 189 dma_async_issue_pending(chan); 190 continue; 191 } 192 193 len -= copy; 194 iov[iovec_idx].iov_len -= copy; 195 iov[iovec_idx].iov_base += copy; 196 197 if (!len) 198 return dma_cookie; 199 200 kdata += copy; 201 iov_byte_offset = 0; 202 page_idx++; 203 } 204 iovec_idx++; 205 } 206 207 /* really bad if we ever run out of iovecs */ 208 BUG(); 209 return -EFAULT; 210} 211 212dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov, 213 struct dma_pinned_list *pinned_list, struct page *page, 214 unsigned int offset, size_t len) 215{ 216 int iov_byte_offset; 217 int copy; 218 dma_cookie_t dma_cookie = 0; 219 int iovec_idx; 220 int page_idx; 221 int err; 222 223 /* this needs as-yet-unimplemented buf-to-buff, so punt. */ 224 /* TODO: use dma for this */ 225 if (!chan || !pinned_list) { 226 u8 *vaddr = kmap(page); 227 err = memcpy_toiovec(iov, vaddr + offset, len); 228 kunmap(page); 229 return err; 230 } 231 232 iovec_idx = 0; 233 while (iovec_idx < pinned_list->nr_iovecs) { 234 struct dma_page_list *page_list; 235 236 /* skip already used-up iovecs */ 237 while (!iov[iovec_idx].iov_len) 238 iovec_idx++; 239 240 page_list = &pinned_list->page_list[iovec_idx]; 241 242 iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); 243 page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) 244 - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; 245 246 /* break up copies to not cross page boundary */ 247 while (iov[iovec_idx].iov_len) { 248 copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); 249 copy = min_t(int, copy, iov[iovec_idx].iov_len); 250 251 dma_cookie = dma_async_memcpy_pg_to_pg(chan, 252 page_list->pages[page_idx], 253 iov_byte_offset, 254 page, 255 offset, 256 copy); 257 /* poll for a descriptor slot */ 258 if (unlikely(dma_cookie < 0)) { 259 dma_async_issue_pending(chan); 260 continue; 261 } 262 263 len -= copy; 264 iov[iovec_idx].iov_len -= copy; 265 iov[iovec_idx].iov_base += copy; 266 267 if (!len) 268 return dma_cookie; 269 270 offset += copy; 271 iov_byte_offset = 0; 272 page_idx++; 273 } 274 iovec_idx++; 275 } 276 277 /* really bad if we ever run out of iovecs */ 278 BUG(); 279 return -EFAULT; 280} 281