mmfile.d revision 1.1.1.1
1// Written in the D programming language. 2 3/** 4 * Read and write memory mapped files. 5 * Copyright: Copyright Digital Mars 2004 - 2009. 6 * License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). 7 * Authors: $(HTTP digitalmars.com, Walter Bright), 8 * Matthew Wilson 9 * Source: $(PHOBOSSRC std/_mmfile.d) 10 * 11 * $(SCRIPT inhibitQuickIndex = 1;) 12 */ 13/* Copyright Digital Mars 2004 - 2009. 14 * Distributed under the Boost Software License, Version 1.0. 15 * (See accompanying file LICENSE_1_0.txt or copy at 16 * http://www.boost.org/LICENSE_1_0.txt) 17 */ 18module std.mmfile; 19 20import core.stdc.errno; 21import core.stdc.stdio; 22import core.stdc.stdlib; 23import std.conv, std.exception, std.stdio; 24import std.file; 25import std.path; 26import std.string; 27 28import std.internal.cstring; 29 30//debug = MMFILE; 31 32version (Windows) 33{ 34 import core.sys.windows.windows; 35 import std.utf; 36 import std.windows.syserror; 37} 38else version (Posix) 39{ 40 import core.sys.posix.fcntl; 41 import core.sys.posix.sys.mman; 42 import core.sys.posix.sys.stat; 43 import core.sys.posix.unistd; 44} 45else 46{ 47 static assert(0); 48} 49 50/** 51 * MmFile objects control the memory mapped file resource. 52 */ 53class MmFile 54{ 55 /** 56 * The mode the memory mapped file is opened with. 57 */ 58 enum Mode 59 { 60 read, /// Read existing file 61 readWriteNew, /// Delete existing file, write new file 62 readWrite, /// Read/Write existing file, create if not existing 63 readCopyOnWrite, /// Read/Write existing file, copy on write 64 } 65 66 /** 67 * Open memory mapped file filename for reading. 68 * File is closed when the object instance is deleted. 69 * Throws: 70 * std.file.FileException 71 */ 72 this(string filename) 73 { 74 this(filename, Mode.read, 0, null); 75 } 76 77 version (linux) this(File file, Mode mode = Mode.read, ulong size = 0, 78 void* address = null, size_t window = 0) 79 { 80 // Save a copy of the File to make sure the fd stays open. 81 this.file = file; 82 this(file.fileno, mode, size, address, window); 83 } 84 85 version (linux) private this(int fildes, Mode mode, ulong size, 86 void* address, size_t window) 87 { 88 int oflag; 89 int fmode; 90 91 switch (mode) 92 { 93 case Mode.read: 94 flags = MAP_SHARED; 95 prot = PROT_READ; 96 oflag = O_RDONLY; 97 fmode = 0; 98 break; 99 100 case Mode.readWriteNew: 101 assert(size != 0); 102 flags = MAP_SHARED; 103 prot = PROT_READ | PROT_WRITE; 104 oflag = O_CREAT | O_RDWR | O_TRUNC; 105 fmode = octal!660; 106 break; 107 108 case Mode.readWrite: 109 flags = MAP_SHARED; 110 prot = PROT_READ | PROT_WRITE; 111 oflag = O_CREAT | O_RDWR; 112 fmode = octal!660; 113 break; 114 115 case Mode.readCopyOnWrite: 116 flags = MAP_PRIVATE; 117 prot = PROT_READ | PROT_WRITE; 118 oflag = O_RDWR; 119 fmode = 0; 120 break; 121 122 default: 123 assert(0); 124 } 125 126 fd = fildes; 127 128 // Adjust size 129 stat_t statbuf = void; 130 errnoEnforce(fstat(fd, &statbuf) == 0); 131 if (prot & PROT_WRITE && size > statbuf.st_size) 132 { 133 // Need to make the file size bytes big 134 lseek(fd, cast(off_t)(size - 1), SEEK_SET); 135 char c = 0; 136 core.sys.posix.unistd.write(fd, &c, 1); 137 } 138 else if (prot & PROT_READ && size == 0) 139 size = statbuf.st_size; 140 this.size = size; 141 142 // Map the file into memory! 143 size_t initial_map = (window && 2*window<size) 144 ? 2*window : cast(size_t) size; 145 auto p = mmap(address, initial_map, prot, flags, fd, 0); 146 if (p == MAP_FAILED) 147 { 148 errnoEnforce(false, "Could not map file into memory"); 149 } 150 data = p[0 .. initial_map]; 151 } 152 153 /** 154 * Open memory mapped file filename in mode. 155 * File is closed when the object instance is deleted. 156 * Params: 157 * filename = name of the file. 158 * If null, an anonymous file mapping is created. 159 * mode = access mode defined above. 160 * size = the size of the file. If 0, it is taken to be the 161 * size of the existing file. 162 * address = the preferred address to map the file to, 163 * although the system is not required to honor it. 164 * If null, the system selects the most convenient address. 165 * window = preferred block size of the amount of data to map at one time 166 * with 0 meaning map the entire file. The window size must be a 167 * multiple of the memory allocation page size. 168 * Throws: 169 * std.file.FileException 170 */ 171 this(string filename, Mode mode, ulong size, void* address, 172 size_t window = 0) 173 { 174 this.filename = filename; 175 this.mMode = mode; 176 this.window = window; 177 this.address = address; 178 179 version (Windows) 180 { 181 void* p; 182 uint dwDesiredAccess2; 183 uint dwShareMode; 184 uint dwCreationDisposition; 185 uint flProtect; 186 187 switch (mode) 188 { 189 case Mode.read: 190 dwDesiredAccess2 = GENERIC_READ; 191 dwShareMode = FILE_SHARE_READ; 192 dwCreationDisposition = OPEN_EXISTING; 193 flProtect = PAGE_READONLY; 194 dwDesiredAccess = FILE_MAP_READ; 195 break; 196 197 case Mode.readWriteNew: 198 assert(size != 0); 199 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE; 200 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; 201 dwCreationDisposition = CREATE_ALWAYS; 202 flProtect = PAGE_READWRITE; 203 dwDesiredAccess = FILE_MAP_WRITE; 204 break; 205 206 case Mode.readWrite: 207 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE; 208 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; 209 dwCreationDisposition = OPEN_ALWAYS; 210 flProtect = PAGE_READWRITE; 211 dwDesiredAccess = FILE_MAP_WRITE; 212 break; 213 214 case Mode.readCopyOnWrite: 215 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE; 216 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; 217 dwCreationDisposition = OPEN_EXISTING; 218 flProtect = PAGE_WRITECOPY; 219 dwDesiredAccess = FILE_MAP_COPY; 220 break; 221 222 default: 223 assert(0); 224 } 225 226 if (filename != null) 227 { 228 hFile = CreateFileW(filename.tempCStringW(), 229 dwDesiredAccess2, 230 dwShareMode, 231 null, 232 dwCreationDisposition, 233 FILE_ATTRIBUTE_NORMAL, 234 cast(HANDLE) null); 235 wenforce(hFile != INVALID_HANDLE_VALUE, "CreateFileW"); 236 } 237 else 238 hFile = INVALID_HANDLE_VALUE; 239 240 scope(failure) 241 { 242 if (hFile != INVALID_HANDLE_VALUE) 243 { 244 CloseHandle(hFile); 245 hFile = INVALID_HANDLE_VALUE; 246 } 247 } 248 249 int hi = cast(int)(size >> 32); 250 hFileMap = CreateFileMappingW(hFile, null, flProtect, 251 hi, cast(uint) size, null); 252 wenforce(hFileMap, "CreateFileMapping"); 253 scope(failure) 254 { 255 CloseHandle(hFileMap); 256 hFileMap = null; 257 } 258 259 if (size == 0 && filename != null) 260 { 261 uint sizehi; 262 uint sizelow = GetFileSize(hFile, &sizehi); 263 wenforce(sizelow != INVALID_FILE_SIZE || GetLastError() != ERROR_SUCCESS, 264 "GetFileSize"); 265 size = (cast(ulong) sizehi << 32) + sizelow; 266 } 267 this.size = size; 268 269 size_t initial_map = (window && 2*window<size) 270 ? 2*window : cast(size_t) size; 271 p = MapViewOfFileEx(hFileMap, dwDesiredAccess, 0, 0, 272 initial_map, address); 273 wenforce(p, "MapViewOfFileEx"); 274 data = p[0 .. initial_map]; 275 276 debug (MMFILE) printf("MmFile.this(): p = %p, size = %d\n", p, size); 277 } 278 else version (Posix) 279 { 280 void* p; 281 int oflag; 282 int fmode; 283 284 switch (mode) 285 { 286 case Mode.read: 287 flags = MAP_SHARED; 288 prot = PROT_READ; 289 oflag = O_RDONLY; 290 fmode = 0; 291 break; 292 293 case Mode.readWriteNew: 294 assert(size != 0); 295 flags = MAP_SHARED; 296 prot = PROT_READ | PROT_WRITE; 297 oflag = O_CREAT | O_RDWR | O_TRUNC; 298 fmode = octal!660; 299 break; 300 301 case Mode.readWrite: 302 flags = MAP_SHARED; 303 prot = PROT_READ | PROT_WRITE; 304 oflag = O_CREAT | O_RDWR; 305 fmode = octal!660; 306 break; 307 308 case Mode.readCopyOnWrite: 309 flags = MAP_PRIVATE; 310 prot = PROT_READ | PROT_WRITE; 311 oflag = O_RDWR; 312 fmode = 0; 313 break; 314 315 default: 316 assert(0); 317 } 318 319 if (filename.length) 320 { 321 fd = .open(filename.tempCString(), oflag, fmode); 322 errnoEnforce(fd != -1, "Could not open file "~filename); 323 324 stat_t statbuf; 325 if (fstat(fd, &statbuf)) 326 { 327 //printf("\tfstat error, errno = %d\n", errno); 328 .close(fd); 329 fd = -1; 330 errnoEnforce(false, "Could not stat file "~filename); 331 } 332 333 if (prot & PROT_WRITE && size > statbuf.st_size) 334 { 335 // Need to make the file size bytes big 336 .lseek(fd, cast(off_t)(size - 1), SEEK_SET); 337 char c = 0; 338 core.sys.posix.unistd.write(fd, &c, 1); 339 } 340 else if (prot & PROT_READ && size == 0) 341 size = statbuf.st_size; 342 } 343 else 344 { 345 fd = -1; 346 version (CRuntime_Glibc) import core.sys.linux.sys.mman : MAP_ANON; 347 flags |= MAP_ANON; 348 } 349 this.size = size; 350 size_t initial_map = (window && 2*window<size) 351 ? 2*window : cast(size_t) size; 352 p = mmap(address, initial_map, prot, flags, fd, 0); 353 if (p == MAP_FAILED) 354 { 355 if (fd != -1) 356 { 357 .close(fd); 358 fd = -1; 359 } 360 errnoEnforce(false, "Could not map file "~filename); 361 } 362 363 data = p[0 .. initial_map]; 364 } 365 else 366 { 367 static assert(0); 368 } 369 } 370 371 /** 372 * Flushes pending output and closes the memory mapped file. 373 */ 374 ~this() 375 { 376 debug (MMFILE) printf("MmFile.~this()\n"); 377 unmap(); 378 data = null; 379 version (Windows) 380 { 381 wenforce(hFileMap == null || CloseHandle(hFileMap) == TRUE, 382 "Could not close file handle"); 383 hFileMap = null; 384 385 wenforce(!hFile || hFile == INVALID_HANDLE_VALUE 386 || CloseHandle(hFile) == TRUE, 387 "Could not close handle"); 388 hFile = INVALID_HANDLE_VALUE; 389 } 390 else version (Posix) 391 { 392 version (linux) 393 { 394 if (file !is File.init) 395 { 396 // The File destructor will close the file, 397 // if it is the only remaining reference. 398 return; 399 } 400 } 401 errnoEnforce(fd == -1 || fd <= 2 402 || .close(fd) != -1, 403 "Could not close handle"); 404 fd = -1; 405 } 406 else 407 { 408 static assert(0); 409 } 410 } 411 412 /* Flush any pending output. 413 */ 414 void flush() 415 { 416 debug (MMFILE) printf("MmFile.flush()\n"); 417 version (Windows) 418 { 419 FlushViewOfFile(data.ptr, data.length); 420 } 421 else version (Posix) 422 { 423 int i; 424 i = msync(cast(void*) data, data.length, MS_SYNC); // sys/mman.h 425 errnoEnforce(i == 0, "msync failed"); 426 } 427 else 428 { 429 static assert(0); 430 } 431 } 432 433 /** 434 * Gives size in bytes of the memory mapped file. 435 */ 436 @property ulong length() const 437 { 438 debug (MMFILE) printf("MmFile.length()\n"); 439 return size; 440 } 441 442 /** 443 * Read-only property returning the file mode. 444 */ 445 Mode mode() 446 { 447 debug (MMFILE) printf("MmFile.mode()\n"); 448 return mMode; 449 } 450 451 /** 452 * Returns entire file contents as an array. 453 */ 454 void[] opSlice() 455 { 456 debug (MMFILE) printf("MmFile.opSlice()\n"); 457 return opSlice(0,size); 458 } 459 460 /** 461 * Returns slice of file contents as an array. 462 */ 463 void[] opSlice(ulong i1, ulong i2) 464 { 465 debug (MMFILE) printf("MmFile.opSlice(%lld, %lld)\n", i1, i2); 466 ensureMapped(i1,i2); 467 size_t off1 = cast(size_t)(i1-start); 468 size_t off2 = cast(size_t)(i2-start); 469 return data[off1 .. off2]; 470 } 471 472 /** 473 * Returns byte at index i in file. 474 */ 475 ubyte opIndex(ulong i) 476 { 477 debug (MMFILE) printf("MmFile.opIndex(%lld)\n", i); 478 ensureMapped(i); 479 size_t off = cast(size_t)(i-start); 480 return (cast(ubyte[]) data)[off]; 481 } 482 483 /** 484 * Sets and returns byte at index i in file to value. 485 */ 486 ubyte opIndexAssign(ubyte value, ulong i) 487 { 488 debug (MMFILE) printf("MmFile.opIndex(%lld, %d)\n", i, value); 489 ensureMapped(i); 490 size_t off = cast(size_t)(i-start); 491 return (cast(ubyte[]) data)[off] = value; 492 } 493 494 495 // return true if the given position is currently mapped 496 private int mapped(ulong i) 497 { 498 debug (MMFILE) printf("MmFile.mapped(%lld, %lld, %d)\n", i,start, 499 data.length); 500 return i >= start && i < start+data.length; 501 } 502 503 // unmap the current range 504 private void unmap() 505 { 506 debug (MMFILE) printf("MmFile.unmap()\n"); 507 version (Windows) 508 { 509 wenforce(!data.ptr || UnmapViewOfFile(data.ptr) != FALSE, "UnmapViewOfFile"); 510 } 511 else 512 { 513 errnoEnforce(!data.ptr || munmap(cast(void*) data, data.length) == 0, 514 "munmap failed"); 515 } 516 data = null; 517 } 518 519 // map range 520 private void map(ulong start, size_t len) 521 { 522 debug (MMFILE) printf("MmFile.map(%lld, %d)\n", start, len); 523 void* p; 524 if (start+len > size) 525 len = cast(size_t)(size-start); 526 version (Windows) 527 { 528 uint hi = cast(uint)(start >> 32); 529 p = MapViewOfFileEx(hFileMap, dwDesiredAccess, hi, cast(uint) start, len, address); 530 wenforce(p, "MapViewOfFileEx"); 531 } 532 else 533 { 534 p = mmap(address, len, prot, flags, fd, cast(off_t) start); 535 errnoEnforce(p != MAP_FAILED); 536 } 537 data = p[0 .. len]; 538 this.start = start; 539 } 540 541 // ensure a given position is mapped 542 private void ensureMapped(ulong i) 543 { 544 debug (MMFILE) printf("MmFile.ensureMapped(%lld)\n", i); 545 if (!mapped(i)) 546 { 547 unmap(); 548 if (window == 0) 549 { 550 map(0,cast(size_t) size); 551 } 552 else 553 { 554 ulong block = i/window; 555 if (block == 0) 556 map(0,2*window); 557 else 558 map(window*(block-1),3*window); 559 } 560 } 561 } 562 563 // ensure a given range is mapped 564 private void ensureMapped(ulong i, ulong j) 565 { 566 debug (MMFILE) printf("MmFile.ensureMapped(%lld, %lld)\n", i, j); 567 if (!mapped(i) || !mapped(j-1)) 568 { 569 unmap(); 570 if (window == 0) 571 { 572 map(0,cast(size_t) size); 573 } 574 else 575 { 576 ulong iblock = i/window; 577 ulong jblock = (j-1)/window; 578 if (iblock == 0) 579 { 580 map(0,cast(size_t)(window*(jblock+2))); 581 } 582 else 583 { 584 map(window*(iblock-1),cast(size_t)(window*(jblock-iblock+3))); 585 } 586 } 587 } 588 } 589 590private: 591 string filename; 592 void[] data; 593 ulong start; 594 size_t window; 595 ulong size; 596 Mode mMode; 597 void* address; 598 version (linux) File file; 599 600 version (Windows) 601 { 602 HANDLE hFile = INVALID_HANDLE_VALUE; 603 HANDLE hFileMap = null; 604 uint dwDesiredAccess; 605 } 606 else version (Posix) 607 { 608 int fd; 609 int prot; 610 int flags; 611 int fmode; 612 } 613 else 614 { 615 static assert(0); 616 } 617 618 // Report error, where errno gives the error number 619 // void errNo() 620 // { 621 // version (Windows) 622 // { 623 // throw new FileException(filename, GetLastError()); 624 // } 625 // else version (linux) 626 // { 627 // throw new FileException(filename, errno); 628 // } 629 // else 630 // { 631 // static assert(0); 632 // } 633 // } 634} 635 636@system unittest 637{ 638 import core.memory : GC; 639 import std.file : deleteme; 640 641 const size_t K = 1024; 642 size_t win = 64*K; // assume the page size is 64K 643 version (Windows) 644 { 645 /+ these aren't defined in core.sys.windows.windows so let's use default 646 SYSTEM_INFO sysinfo; 647 GetSystemInfo(&sysinfo); 648 win = sysinfo.dwAllocationGranularity; 649 +/ 650 } 651 else version (linux) 652 { 653 // getpagesize() is not defined in the unix D headers so use the guess 654 } 655 string test_file = std.file.deleteme ~ "-testing.txt"; 656 MmFile mf = new MmFile(test_file,MmFile.Mode.readWriteNew, 657 100*K,null,win); 658 ubyte[] str = cast(ubyte[])"1234567890"; 659 ubyte[] data = cast(ubyte[]) mf[0 .. 10]; 660 data[] = str[]; 661 assert( mf[0 .. 10] == str ); 662 data = cast(ubyte[]) mf[50 .. 60]; 663 data[] = str[]; 664 assert( mf[50 .. 60] == str ); 665 ubyte[] data2 = cast(ubyte[]) mf[20*K .. 60*K]; 666 assert( data2.length == 40*K ); 667 assert( data2[$-1] == 0 ); 668 mf[100*K-1] = cast(ubyte)'b'; 669 data2 = cast(ubyte[]) mf[21*K .. 100*K]; 670 assert( data2.length == 79*K ); 671 assert( data2[$-1] == 'b' ); 672 673 destroy(mf); 674 GC.free(&mf); 675 676 std.file.remove(test_file); 677 // Create anonymous mapping 678 auto test = new MmFile(null, MmFile.Mode.readWriteNew, 1024*1024, null); 679} 680 681version (linux) 682@system unittest // Issue 14868 683{ 684 import std.file : deleteme; 685 import std.typecons : scoped; 686 687 // Test retaining ownership of File/fd 688 689 auto fn = std.file.deleteme ~ "-testing.txt"; 690 scope(exit) std.file.remove(fn); 691 File(fn, "wb").writeln("Testing!"); 692 scoped!MmFile(File(fn)); 693 694 // Test that unique ownership of File actually leads to the fd being closed 695 696 auto f = File(fn); 697 auto fd = f.fileno; 698 { 699 auto mf = scoped!MmFile(f); 700 f = File.init; 701 } 702 assert(.close(fd) == -1); 703} 704 705@system unittest // Issue 14994, 14995 706{ 707 import std.file : deleteme; 708 import std.typecons : scoped; 709 710 // Zero-length map may or may not be valid on OSX and NetBSD 711 version (OSX) 712 import std.exception : verifyThrown = collectException; 713 version (NetBSD) 714 import std.exception : verifyThrown = collectException; 715 else 716 import std.exception : verifyThrown = assertThrown; 717 718 auto fn = std.file.deleteme ~ "-testing.txt"; 719 scope(exit) std.file.remove(fn); 720 verifyThrown(scoped!MmFile(fn, MmFile.Mode.readWrite, 0, null)); 721} 722