1/* 2 * Copyright (c) 2009 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28#include <mach_assert.h> 29 30#include <sys/errno.h> 31#include <i386/param.h> 32#include <i386/misc_protos.h> 33#include <i386/cpu_data.h> 34#include <i386/machine_routines.h> 35#include <i386/cpuid.h> 36#include <i386/vmx.h> 37#include <vm/pmap.h> 38#include <vm/vm_map.h> 39#include <vm/vm_kern.h> 40#include <vm/vm_fault.h> 41 42#include <sys/kdebug.h> 43 44static int copyio(int, user_addr_t, char *, vm_size_t, vm_size_t *, int); 45static int copyio_phys(addr64_t, addr64_t, vm_size_t, int); 46 47/* 48 * The copy engine has the following characteristics 49 * - copyio() handles copies to/from user or kernel space 50 * - copypv() deals with physical or virtual addresses 51 * 52 * Readers familiar with the 32-bit kernel will expect Joe's thesis at this 53 * point describing the full glory of the copy window implementation. In K64, 54 * however, there is no need for windowing. Thanks to the vast shared address 55 * space, the kernel has direct access to userspace and to physical memory. 56 * 57 * User virtual addresses are accessible provided the user's cr3 is loaded. 58 * Physical addresses are accessible via the direct map and the PHYSMAP_PTOV() 59 * translation. 60 * 61 * Copyin/out variants all boil done to just these 2 routines in locore.s which 62 * provide fault-recoverable copying: 63 */ 64extern int _bcopy(const void *, void *, vm_size_t); 65extern int _bcopystr(const void *, void *, vm_size_t, vm_size_t *); 66 67 68/* 69 * Types of copies: 70 */ 71#define COPYIN 0 /* from user virtual to kernel virtual */ 72#define COPYOUT 1 /* from kernel virtual to user virtual */ 73#define COPYINSTR 2 /* string variant of copyout */ 74#define COPYINPHYS 3 /* from user virtual to kernel physical */ 75#define COPYOUTPHYS 4 /* from kernel physical to user virtual */ 76 77static int 78copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, 79 vm_size_t nbytes, vm_size_t *lencopied, int use_kernel_map) 80{ 81 thread_t thread; 82 pmap_t pmap; 83 vm_size_t bytes_copied; 84 int error = 0; 85 boolean_t istate = FALSE; 86 boolean_t recursive_CopyIOActive; 87#if KDEBUG 88 int debug_type = 0xeff70010; 89 debug_type += (copy_type << 2); 90#endif 91 92 thread = current_thread(); 93 94 KERNEL_DEBUG(debug_type | DBG_FUNC_START, 95 (unsigned)(user_addr >> 32), (unsigned)user_addr, 96 nbytes, thread->machine.copyio_state, 0); 97 98 if (nbytes == 0) 99 goto out; 100 101 pmap = thread->map->pmap; 102 103 if ((copy_type != COPYINPHYS) && (copy_type != COPYOUTPHYS) && ((vm_offset_t)kernel_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS)) { 104 panic("Invalid copy parameter, copy type: %d, kernel address: %p", copy_type, kernel_addr); 105 } 106 107 /* Sanity and security check for addresses to/from a user */ 108 109 if (((pmap != kernel_pmap) && (use_kernel_map == 0)) && 110 ((nbytes && (user_addr+nbytes <= user_addr)) || ((user_addr + nbytes) > vm_map_max(thread->map)))) { 111 error = EFAULT; 112 goto out; 113 } 114 115 /* 116 * If the no_shared_cr3 boot-arg is set (true), the kernel runs on 117 * its own pmap and cr3 rather than the user's -- so that wild accesses 118 * from kernel or kexts can be trapped. So, during copyin and copyout, 119 * we need to switch back to the user's map/cr3. The thread is flagged 120 * "CopyIOActive" at this time so that if the thread is pre-empted, 121 * we will later restore the correct cr3. 122 */ 123 recursive_CopyIOActive = thread->machine.specFlags & CopyIOActive; 124 thread->machine.specFlags |= CopyIOActive; 125 if (no_shared_cr3) { 126 istate = ml_set_interrupts_enabled(FALSE); 127 if (get_cr3_base() != pmap->pm_cr3) 128 set_cr3_raw(pmap->pm_cr3); 129 } 130 131 /* 132 * Ensure that we're running on the target thread's cr3. 133 */ 134 if ((pmap != kernel_pmap) && !use_kernel_map && 135 (get_cr3_base() != pmap->pm_cr3)) { 136 panic("copyio(%d,%p,%p,%ld,%p,%d) cr3 is %p expects %p", 137 copy_type, (void *)user_addr, kernel_addr, nbytes, lencopied, use_kernel_map, 138 (void *) get_cr3_raw(), (void *) pmap->pm_cr3); 139 } 140 if (no_shared_cr3) 141 (void) ml_set_interrupts_enabled(istate); 142 143 KERNEL_DEBUG(0xeff70044 | DBG_FUNC_NONE, (unsigned)user_addr, 144 (unsigned)kernel_addr, nbytes, 0, 0); 145 146 switch (copy_type) { 147 148 case COPYIN: 149 error = _bcopy((const void *) user_addr, 150 kernel_addr, 151 nbytes); 152 break; 153 154 case COPYOUT: 155 error = _bcopy(kernel_addr, 156 (void *) user_addr, 157 nbytes); 158 break; 159 160 case COPYINPHYS: 161 error = _bcopy((const void *) user_addr, 162 PHYSMAP_PTOV(kernel_addr), 163 nbytes); 164 break; 165 166 case COPYOUTPHYS: 167 error = _bcopy((const void *) PHYSMAP_PTOV(kernel_addr), 168 (void *) user_addr, 169 nbytes); 170 break; 171 172 case COPYINSTR: 173 error = _bcopystr((const void *) user_addr, 174 kernel_addr, 175 (int) nbytes, 176 &bytes_copied); 177 178 /* 179 * lencopied should be updated on success 180 * or ENAMETOOLONG... but not EFAULT 181 */ 182 if (error != EFAULT) 183 *lencopied = bytes_copied; 184 185 if (error) { 186#if KDEBUG 187 nbytes = *lencopied; 188#endif 189 break; 190 } 191 if (*(kernel_addr + bytes_copied - 1) == 0) { 192 /* 193 * we found a NULL terminator... we're done 194 */ 195#if KDEBUG 196 nbytes = *lencopied; 197#endif 198 break; 199 } else { 200 /* 201 * no more room in the buffer and we haven't 202 * yet come across a NULL terminator 203 */ 204#if KDEBUG 205 nbytes = *lencopied; 206#endif 207 error = ENAMETOOLONG; 208 break; 209 } 210 break; 211 } 212 213 if (!recursive_CopyIOActive) { 214 thread->machine.specFlags &= ~CopyIOActive; 215 } 216 if (no_shared_cr3) { 217 istate = ml_set_interrupts_enabled(FALSE); 218 if (get_cr3_raw() != kernel_pmap->pm_cr3) 219 set_cr3_raw(kernel_pmap->pm_cr3); 220 (void) ml_set_interrupts_enabled(istate); 221 } 222 223out: 224 KERNEL_DEBUG(debug_type | DBG_FUNC_END, (unsigned)user_addr, 225 (unsigned)kernel_addr, (unsigned)nbytes, error, 0); 226 227 return (error); 228} 229 230 231static int 232copyio_phys(addr64_t source, addr64_t sink, vm_size_t csize, int which) 233{ 234 char *paddr; 235 user_addr_t vaddr; 236 int ctype; 237 238 if (which & cppvPsnk) { 239 paddr = (char *)sink; 240 vaddr = (user_addr_t)source; 241 ctype = COPYINPHYS; 242 } else { 243 paddr = (char *)source; 244 vaddr = (user_addr_t)sink; 245 ctype = COPYOUTPHYS; 246 } 247 return copyio(ctype, vaddr, paddr, csize, NULL, which & cppvKmap); 248} 249 250int 251copyinmsg(const user_addr_t user_addr, char *kernel_addr, mach_msg_size_t nbytes) 252{ 253 return copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0); 254} 255 256int 257copyin(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes) 258{ 259 return copyio(COPYIN, user_addr, kernel_addr, nbytes, NULL, 0); 260} 261 262int 263copyinstr(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied) 264{ 265 *lencopied = 0; 266 267 return copyio(COPYINSTR, user_addr, kernel_addr, nbytes, lencopied, 0); 268} 269 270int 271copyoutmsg(const char *kernel_addr, user_addr_t user_addr, mach_msg_size_t nbytes) 272{ 273 return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0); 274} 275 276int 277copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes) 278{ 279 return copyio(COPYOUT, user_addr, (char *)(uintptr_t)kernel_addr, nbytes, NULL, 0); 280} 281 282 283kern_return_t 284copypv(addr64_t src64, addr64_t snk64, unsigned int size, int which) 285{ 286 unsigned int lop, csize; 287 int bothphys = 0; 288 289 KERNEL_DEBUG(0xeff7004c | DBG_FUNC_START, (unsigned)src64, 290 (unsigned)snk64, size, which, 0); 291 292 if ((which & (cppvPsrc | cppvPsnk)) == 0 ) /* Make sure that only one is virtual */ 293 panic("copypv: no more than 1 parameter may be virtual\n"); /* Not allowed */ 294 295 if ((which & (cppvPsrc | cppvPsnk)) == (cppvPsrc | cppvPsnk)) 296 bothphys = 1; /* both are physical */ 297 298 while (size) { 299 300 if (bothphys) { 301 lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1))); /* Assume sink smallest */ 302 303 if (lop > (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1)))) 304 lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1))); /* No, source is smaller */ 305 } else { 306 /* 307 * only need to compute the resid for the physical page 308 * address... we don't care about where we start/finish in 309 * the virtual since we just call the normal copyin/copyout 310 */ 311 if (which & cppvPsrc) 312 lop = (unsigned int)(PAGE_SIZE - (src64 & (PAGE_SIZE - 1))); 313 else 314 lop = (unsigned int)(PAGE_SIZE - (snk64 & (PAGE_SIZE - 1))); 315 } 316 csize = size; /* Assume we can copy it all */ 317 if (lop < size) 318 csize = lop; /* Nope, we can't do it all */ 319#if 0 320 /* 321 * flush_dcache64 is currently a nop on the i386... 322 * it's used when copying to non-system memory such 323 * as video capture cards... on PPC there was a need 324 * to flush due to how we mapped this memory... not 325 * sure if it's needed on i386. 326 */ 327 if (which & cppvFsrc) 328 flush_dcache64(src64, csize, 1); /* If requested, flush source before move */ 329 if (which & cppvFsnk) 330 flush_dcache64(snk64, csize, 1); /* If requested, flush sink before move */ 331#endif 332 if (bothphys) 333 bcopy_phys(src64, snk64, csize); /* Do a physical copy, virtually */ 334 else { 335 if (copyio_phys(src64, snk64, csize, which)) 336 return (KERN_FAILURE); 337 } 338#if 0 339 if (which & cppvFsrc) 340 flush_dcache64(src64, csize, 1); /* If requested, flush source after move */ 341 if (which & cppvFsnk) 342 flush_dcache64(snk64, csize, 1); /* If requested, flush sink after move */ 343#endif 344 size -= csize; /* Calculate what is left */ 345 snk64 += csize; /* Bump sink to next physical address */ 346 src64 += csize; /* Bump source to next physical address */ 347 } 348 KERNEL_DEBUG(0xeff7004c | DBG_FUNC_END, (unsigned)src64, 349 (unsigned)snk64, size, which, 0); 350 351 return KERN_SUCCESS; 352} 353