1// Written in the D programming language.
2
3/**
4 * Read and write memory mapped files.
5 * Copyright: Copyright The D Language Foundation 2004 - 2009.
6 * License:   $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
7 * Authors:   $(HTTP digitalmars.com, Walter Bright),
8 *            Matthew Wilson
9 * Source:    $(PHOBOSSRC std/mmfile.d)
10 *
11 * $(SCRIPT inhibitQuickIndex = 1;)
12 */
13/*          Copyright The D Language Foundation 2004 - 2009.
14 * Distributed under the Boost Software License, Version 1.0.
15 *    (See accompanying file LICENSE_1_0.txt or copy at
16 *          http://www.boost.org/LICENSE_1_0.txt)
17 */
18module std.mmfile;
19
20import core.stdc.errno;
21import core.stdc.stdio;
22import core.stdc.stdlib;
23import std.conv, std.exception, std.stdio;
24import std.file;
25import std.path;
26import std.string;
27
28import std.internal.cstring;
29
30//debug = MMFILE;
31
32version (Windows)
33{
34    import core.sys.windows.winbase;
35    import core.sys.windows.winnt;
36    import std.utf;
37    import std.windows.syserror;
38}
39else version (Posix)
40{
41    import core.sys.posix.fcntl;
42    import core.sys.posix.sys.mman;
43    import core.sys.posix.sys.stat;
44    import core.sys.posix.unistd;
45}
46else
47{
48    static assert(0);
49}
50
51/**
52 * MmFile objects control the memory mapped file resource.
53 */
54class MmFile
55{
56    /**
57     * The mode the memory mapped file is opened with.
58     */
59    enum Mode
60    {
61        read,            /// Read existing file
62        readWriteNew,    /// Delete existing file, write new file
63        readWrite,       /// Read/Write existing file, create if not existing
64        readCopyOnWrite, /// Read/Write existing file, copy on write
65    }
66
67    /**
68     * Open memory mapped file filename for reading.
69     * File is closed when the object instance is deleted.
70     * Throws:
71     *  - On POSIX, $(REF ErrnoException, std, exception).
72     *  - On Windows, $(REF WindowsException, std, windows, syserror).
73     */
74    this(string filename)
75    {
76        this(filename, Mode.read, 0, null);
77    }
78
79    version (linux) this(File file, Mode mode = Mode.read, ulong size = 0,
80            void* address = null, size_t window = 0)
81    {
82        // Save a copy of the File to make sure the fd stays open.
83        this.file = file;
84        this(file.fileno, mode, size, address, window);
85    }
86
87    version (linux) private this(int fildes, Mode mode, ulong size,
88            void* address, size_t window)
89    {
90        int oflag;
91        int fmode;
92
93        final switch (mode)
94        {
95        case Mode.read:
96            flags = MAP_SHARED;
97            prot = PROT_READ;
98            oflag = O_RDONLY;
99            fmode = 0;
100            break;
101
102        case Mode.readWriteNew:
103            assert(size != 0);
104            flags = MAP_SHARED;
105            prot = PROT_READ | PROT_WRITE;
106            oflag = O_CREAT | O_RDWR | O_TRUNC;
107            fmode = octal!660;
108            break;
109
110        case Mode.readWrite:
111            flags = MAP_SHARED;
112            prot = PROT_READ | PROT_WRITE;
113            oflag = O_CREAT | O_RDWR;
114            fmode = octal!660;
115            break;
116
117        case Mode.readCopyOnWrite:
118            flags = MAP_PRIVATE;
119            prot = PROT_READ | PROT_WRITE;
120            oflag = O_RDWR;
121            fmode = 0;
122            break;
123        }
124
125        fd = fildes;
126
127        // Adjust size
128        stat_t statbuf = void;
129        errnoEnforce(fstat(fd, &statbuf) == 0);
130        if (prot & PROT_WRITE && size > statbuf.st_size)
131        {
132            // Need to make the file size bytes big
133            lseek(fd, cast(off_t)(size - 1), SEEK_SET);
134            char c = 0;
135            core.sys.posix.unistd.write(fd, &c, 1);
136        }
137        else if (prot & PROT_READ && size == 0)
138            size = statbuf.st_size;
139        this.size = size;
140
141        // Map the file into memory!
142        size_t initial_map = (window && 2*window<size)
143            ? 2*window : cast(size_t) size;
144        auto p = mmap(address, initial_map, prot, flags, fd, 0);
145        if (p == MAP_FAILED)
146        {
147            errnoEnforce(false, "Could not map file into memory");
148        }
149        data = p[0 .. initial_map];
150    }
151
152    /**
153     * Open memory mapped file filename in mode.
154     * File is closed when the object instance is deleted.
155     * Params:
156     *  filename = name of the file.
157     *      If null, an anonymous file mapping is created.
158     *  mode = access mode defined above.
159     *  size =  the size of the file. If 0, it is taken to be the
160     *      size of the existing file.
161     *  address = the preferred address to map the file to,
162     *      although the system is not required to honor it.
163     *      If null, the system selects the most convenient address.
164     *  window = preferred block size of the amount of data to map at one time
165     *      with 0 meaning map the entire file. The window size must be a
166     *      multiple of the memory allocation page size.
167     * Throws:
168     *  - On POSIX, $(REF ErrnoException, std, exception).
169     *  - On Windows, $(REF WindowsException, std, windows, syserror).
170     */
171    this(string filename, Mode mode, ulong size, void* address,
172            size_t window = 0)
173    {
174        this.filename = filename;
175        this.mMode = mode;
176        this.window = window;
177        this.address = address;
178
179        version (Windows)
180        {
181            void* p;
182            uint dwDesiredAccess2;
183            uint dwShareMode;
184            uint dwCreationDisposition;
185            uint flProtect;
186
187            final switch (mode)
188            {
189            case Mode.read:
190                dwDesiredAccess2 = GENERIC_READ;
191                dwShareMode = FILE_SHARE_READ;
192                dwCreationDisposition = OPEN_EXISTING;
193                flProtect = PAGE_READONLY;
194                dwDesiredAccess = FILE_MAP_READ;
195                break;
196
197            case Mode.readWriteNew:
198                assert(size != 0);
199                dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
200                dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
201                dwCreationDisposition = CREATE_ALWAYS;
202                flProtect = PAGE_READWRITE;
203                dwDesiredAccess = FILE_MAP_WRITE;
204                break;
205
206            case Mode.readWrite:
207                dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
208                dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
209                dwCreationDisposition = OPEN_ALWAYS;
210                flProtect = PAGE_READWRITE;
211                dwDesiredAccess = FILE_MAP_WRITE;
212                break;
213
214            case Mode.readCopyOnWrite:
215                dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
216                dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
217                dwCreationDisposition = OPEN_EXISTING;
218                flProtect = PAGE_WRITECOPY;
219                dwDesiredAccess = FILE_MAP_COPY;
220                break;
221            }
222
223            if (filename != null)
224            {
225                hFile = CreateFileW(filename.tempCStringW(),
226                        dwDesiredAccess2,
227                        dwShareMode,
228                        null,
229                        dwCreationDisposition,
230                        FILE_ATTRIBUTE_NORMAL,
231                        cast(HANDLE) null);
232                wenforce(hFile != INVALID_HANDLE_VALUE, "CreateFileW");
233            }
234            else
235                hFile = INVALID_HANDLE_VALUE;
236
237            scope(failure)
238            {
239                if (hFile != INVALID_HANDLE_VALUE)
240                {
241                    CloseHandle(hFile);
242                    hFile = INVALID_HANDLE_VALUE;
243                }
244            }
245
246            int hi = cast(int)(size >> 32);
247            hFileMap = CreateFileMappingW(hFile, null, flProtect,
248                    hi, cast(uint) size, null);
249            wenforce(hFileMap, "CreateFileMapping");
250            scope(failure)
251            {
252                CloseHandle(hFileMap);
253                hFileMap = null;
254            }
255
256            if (size == 0 && filename != null)
257            {
258                uint sizehi;
259                uint sizelow = GetFileSize(hFile, &sizehi);
260                wenforce(sizelow != INVALID_FILE_SIZE || GetLastError() != ERROR_SUCCESS,
261                    "GetFileSize");
262                size = (cast(ulong) sizehi << 32) + sizelow;
263            }
264            this.size = size;
265
266            size_t initial_map = (window && 2*window<size)
267                ? 2*window : cast(size_t) size;
268            p = MapViewOfFileEx(hFileMap, dwDesiredAccess, 0, 0,
269                    initial_map, address);
270            wenforce(p, "MapViewOfFileEx");
271            data = p[0 .. initial_map];
272
273            debug (MMFILE) printf("MmFile.this(): p = %p, size = %d\n", p, size);
274        }
275        else version (Posix)
276        {
277            void* p;
278            int oflag;
279            int fmode;
280
281            final switch (mode)
282            {
283            case Mode.read:
284                flags = MAP_SHARED;
285                prot = PROT_READ;
286                oflag = O_RDONLY;
287                fmode = 0;
288                break;
289
290            case Mode.readWriteNew:
291                assert(size != 0);
292                flags = MAP_SHARED;
293                prot = PROT_READ | PROT_WRITE;
294                oflag = O_CREAT | O_RDWR | O_TRUNC;
295                fmode = octal!660;
296                break;
297
298            case Mode.readWrite:
299                flags = MAP_SHARED;
300                prot = PROT_READ | PROT_WRITE;
301                oflag = O_CREAT | O_RDWR;
302                fmode = octal!660;
303                break;
304
305            case Mode.readCopyOnWrite:
306                flags = MAP_PRIVATE;
307                prot = PROT_READ | PROT_WRITE;
308                oflag = O_RDWR;
309                fmode = 0;
310                break;
311            }
312
313            if (filename.length)
314            {
315                fd = .open(filename.tempCString(), oflag, fmode);
316                errnoEnforce(fd != -1, "Could not open file "~filename);
317
318                stat_t statbuf;
319                if (fstat(fd, &statbuf))
320                {
321                    //printf("\tfstat error, errno = %d\n", errno);
322                    .close(fd);
323                    fd = -1;
324                    errnoEnforce(false, "Could not stat file "~filename);
325                }
326
327                if (prot & PROT_WRITE && size > statbuf.st_size)
328                {
329                    // Need to make the file size bytes big
330                    .lseek(fd, cast(off_t)(size - 1), SEEK_SET);
331                    char c = 0;
332                    core.sys.posix.unistd.write(fd, &c, 1);
333                }
334                else if (prot & PROT_READ && size == 0)
335                    size = statbuf.st_size;
336            }
337            else
338            {
339                fd = -1;
340                flags |= MAP_ANON;
341            }
342            this.size = size;
343            size_t initial_map = (window && 2*window<size)
344                ? 2*window : cast(size_t) size;
345            p = mmap(address, initial_map, prot, flags, fd, 0);
346            if (p == MAP_FAILED)
347            {
348                if (fd != -1)
349                {
350                    .close(fd);
351                    fd = -1;
352                }
353                errnoEnforce(false, "Could not map file "~filename);
354            }
355
356            data = p[0 .. initial_map];
357        }
358        else
359        {
360            static assert(0);
361        }
362    }
363
364    /**
365     * Flushes pending output and closes the memory mapped file.
366     */
367    ~this()
368    {
369        debug (MMFILE) printf("MmFile.~this()\n");
370        unmap();
371        data = null;
372        version (Windows)
373        {
374            wenforce(hFileMap == null || CloseHandle(hFileMap) == TRUE,
375                    "Could not close file handle");
376            hFileMap = null;
377
378            wenforce(!hFile || hFile == INVALID_HANDLE_VALUE
379                    || CloseHandle(hFile) == TRUE,
380                    "Could not close handle");
381            hFile = INVALID_HANDLE_VALUE;
382        }
383        else version (Posix)
384        {
385            version (linux)
386            {
387                if (file !is File.init)
388                {
389                    // The File destructor will close the file,
390                    // if it is the only remaining reference.
391                    return;
392                }
393            }
394            errnoEnforce(fd == -1 || fd <= 2
395                    || .close(fd) != -1,
396                    "Could not close handle");
397            fd = -1;
398        }
399        else
400        {
401            static assert(0);
402        }
403    }
404
405    /* Flush any pending output.
406     */
407    void flush()
408    {
409        debug (MMFILE) printf("MmFile.flush()\n");
410        version (Windows)
411        {
412            FlushViewOfFile(data.ptr, data.length);
413        }
414        else version (Posix)
415        {
416            int i;
417            i = msync(cast(void*) data, data.length, MS_SYNC);   // sys/mman.h
418            errnoEnforce(i == 0, "msync failed");
419        }
420        else
421        {
422            static assert(0);
423        }
424    }
425
426    /**
427     * Gives size in bytes of the memory mapped file.
428     */
429    @property ulong length() const
430    {
431        debug (MMFILE) printf("MmFile.length()\n");
432        return size;
433    }
434
435    /**
436     * Forwards `length`.
437     */
438    alias opDollar = length;
439
440    /**
441     * Read-only property returning the file mode.
442     */
443    Mode mode()
444    {
445        debug (MMFILE) printf("MmFile.mode()\n");
446        return mMode;
447    }
448
449    /**
450     * Returns entire file contents as an array.
451     */
452    void[] opSlice()
453    {
454        debug (MMFILE) printf("MmFile.opSlice()\n");
455        return opSlice(0,size);
456    }
457
458    /**
459     * Returns slice of file contents as an array.
460     */
461    void[] opSlice(ulong i1, ulong i2)
462    {
463        debug (MMFILE) printf("MmFile.opSlice(%lld, %lld)\n", i1, i2);
464        ensureMapped(i1,i2);
465        size_t off1 = cast(size_t)(i1-start);
466        size_t off2 = cast(size_t)(i2-start);
467        return data[off1 .. off2];
468    }
469
470    /**
471     * Returns byte at index i in file.
472     */
473    ubyte opIndex(ulong i)
474    {
475        debug (MMFILE) printf("MmFile.opIndex(%lld)\n", i);
476        ensureMapped(i);
477        size_t off = cast(size_t)(i-start);
478        return (cast(ubyte[]) data)[off];
479    }
480
481    /**
482     * Sets and returns byte at index i in file to value.
483     */
484    ubyte opIndexAssign(ubyte value, ulong i)
485    {
486        debug (MMFILE) printf("MmFile.opIndex(%lld, %d)\n", i, value);
487        ensureMapped(i);
488        size_t off = cast(size_t)(i-start);
489        return (cast(ubyte[]) data)[off] = value;
490    }
491
492
493    // return true if the given position is currently mapped
494    private int mapped(ulong i)
495    {
496        debug (MMFILE) printf("MmFile.mapped(%lld, %lld, %d)\n", i,start,
497                data.length);
498        return i >= start && i < start+data.length;
499    }
500
501    // unmap the current range
502    private void unmap()
503    {
504        debug (MMFILE) printf("MmFile.unmap()\n");
505        version (Windows)
506        {
507            wenforce(!data.ptr || UnmapViewOfFile(data.ptr) != FALSE, "UnmapViewOfFile");
508        }
509        else
510        {
511            errnoEnforce(!data.ptr || munmap(cast(void*) data, data.length) == 0,
512                    "munmap failed");
513        }
514        data = null;
515    }
516
517    // map range
518    private void map(ulong start, size_t len)
519    {
520        debug (MMFILE) printf("MmFile.map(%lld, %d)\n", start, len);
521        void* p;
522        if (start+len > size)
523            len = cast(size_t)(size-start);
524        version (Windows)
525        {
526            uint hi = cast(uint)(start >> 32);
527            p = MapViewOfFileEx(hFileMap, dwDesiredAccess, hi, cast(uint) start, len, address);
528            wenforce(p, "MapViewOfFileEx");
529        }
530        else
531        {
532            p = mmap(address, len, prot, flags, fd, cast(off_t) start);
533            errnoEnforce(p != MAP_FAILED);
534        }
535        data = p[0 .. len];
536        this.start = start;
537    }
538
539    // ensure a given position is mapped
540    private void ensureMapped(ulong i)
541    {
542        debug (MMFILE) printf("MmFile.ensureMapped(%lld)\n", i);
543        if (!mapped(i))
544        {
545            unmap();
546            if (window == 0)
547            {
548                map(0,cast(size_t) size);
549            }
550            else
551            {
552                ulong block = i/window;
553                if (block == 0)
554                    map(0,2*window);
555                else
556                    map(window*(block-1),3*window);
557            }
558        }
559    }
560
561    // ensure a given range is mapped
562    private void ensureMapped(ulong i, ulong j)
563    {
564        debug (MMFILE) printf("MmFile.ensureMapped(%lld, %lld)\n", i, j);
565        if (!mapped(i) || !mapped(j-1))
566        {
567            unmap();
568            if (window == 0)
569            {
570                map(0,cast(size_t) size);
571            }
572            else
573            {
574                ulong iblock = i/window;
575                ulong jblock = (j-1)/window;
576                if (iblock == 0)
577                {
578                    map(0,cast(size_t)(window*(jblock+2)));
579                }
580                else
581                {
582                    map(window*(iblock-1),cast(size_t)(window*(jblock-iblock+3)));
583                }
584            }
585        }
586    }
587
588private:
589    string filename;
590    void[] data;
591    ulong  start;
592    size_t window;
593    ulong  size;
594    Mode   mMode;
595    void*  address;
596    version (linux) File file;
597
598    version (Windows)
599    {
600        HANDLE hFile = INVALID_HANDLE_VALUE;
601        HANDLE hFileMap = null;
602        uint dwDesiredAccess;
603    }
604    else version (Posix)
605    {
606        int fd;
607        int prot;
608        int flags;
609        int fmode;
610    }
611    else
612    {
613        static assert(0);
614    }
615
616    // Report error, where errno gives the error number
617    // void errNo()
618    // {
619    //     version (Windows)
620    //     {
621    //         throw new FileException(filename, GetLastError());
622    //     }
623    //     else version (linux)
624    //     {
625    //         throw new FileException(filename, errno);
626    //     }
627    //     else
628    //     {
629    //         static assert(0);
630    //     }
631    // }
632}
633
634@system unittest
635{
636    import core.memory : GC;
637    import std.file : deleteme;
638
639    const size_t K = 1024;
640    size_t win = 64*K; // assume the page size is 64K
641    version (Windows)
642    {
643        /+ these aren't defined in core.sys.windows.windows so let's use default
644         SYSTEM_INFO sysinfo;
645         GetSystemInfo(&sysinfo);
646         win = sysinfo.dwAllocationGranularity;
647         +/
648    }
649    else version (Posix)
650    {
651        import core.sys.posix.unistd;
652        win = cast(size_t) sysconf(_SC_PAGESIZE);
653    }
654    string test_file = std.file.deleteme ~ "-testing.txt";
655    MmFile mf = new MmFile(test_file,MmFile.Mode.readWriteNew,
656            100*K,null,win);
657    ubyte[] str = cast(ubyte[])"1234567890";
658    ubyte[] data = cast(ubyte[]) mf[0 .. 10];
659    data[] = str[];
660    assert( mf[0 .. 10] == str );
661    data = cast(ubyte[]) mf[50 .. 60];
662    data[] = str[];
663    assert( mf[50 .. 60] == str );
664    ubyte[] data2 = cast(ubyte[]) mf[20*K .. 60*K];
665    assert( data2.length == 40*K );
666    assert( data2[$-1] == 0 );
667    mf[100*K-1] = cast(ubyte)'b';
668    data2 = cast(ubyte[]) mf[21*K .. 100*K];
669    assert( data2.length == 79*K );
670    assert( data2[$-1] == 'b' );
671
672    destroy(mf);
673
674    std.file.remove(test_file);
675    // Create anonymous mapping
676    auto test = new MmFile(null, MmFile.Mode.readWriteNew, 1024*1024, null);
677}
678
679version (linux)
680@system unittest // https://issues.dlang.org/show_bug.cgi?id=14868
681{
682    import std.file : deleteme;
683    import std.typecons : scoped;
684
685    // Test retaining ownership of File/fd
686
687    auto fn = std.file.deleteme ~ "-testing.txt";
688    scope(exit) std.file.remove(fn);
689    File(fn, "wb").writeln("Testing!");
690    scoped!MmFile(File(fn));
691
692    // Test that unique ownership of File actually leads to the fd being closed
693
694    auto f = File(fn);
695    auto fd = f.fileno;
696    {
697        auto mf = scoped!MmFile(f);
698        f = File.init;
699    }
700    assert(.close(fd) == -1);
701}
702
703// https://issues.dlang.org/show_bug.cgi?id=14994
704// https://issues.dlang.org/show_bug.cgi?id=14995
705@system unittest
706{
707    import std.file : deleteme;
708    import std.typecons : scoped;
709
710    // Zero-length map may or may not be valid on OSX and NetBSD
711    version (OSX)
712        import std.exception : verifyThrown = collectException;
713    version (NetBSD)
714        import std.exception : verifyThrown = collectException;
715    else
716        import std.exception : verifyThrown = assertThrown;
717
718    auto fn = std.file.deleteme ~ "-testing.txt";
719    scope(exit) std.file.remove(fn);
720    verifyThrown(scoped!MmFile(fn, MmFile.Mode.readWrite, 0, null));
721}
722
723@system unittest
724{
725    MmFile shar = new MmFile(null, MmFile.Mode.readWrite, 10, null, 0);
726    void[] output = shar[0 .. $];
727}
728
729@system unittest
730{
731    import std.file : deleteme;
732    auto name = std.file.deleteme ~ "-test.tmp";
733    scope(exit) std.file.remove(name);
734
735    std.file.write(name, "abcd");
736    {
737        scope MmFile mmf = new MmFile(name);
738        string p;
739
740        assert(mmf[0] == 'a');
741        p = cast(string) mmf[];
742        assert(p[1] == 'b');
743        p = cast(string) mmf[0 .. 4];
744        assert(p[2] == 'c');
745    }
746    {
747        scope MmFile mmf = new MmFile(name, MmFile.Mode.read, 0, null);
748        string p;
749
750        assert(mmf[0] == 'a');
751        p = cast(string) mmf[];
752        assert(mmf.length == 4);
753        assert(p[1] == 'b');
754        p = cast(string) mmf[0 .. 4];
755        assert(p[2] == 'c');
756    }
757    std.file.remove(name);
758    {
759        scope MmFile mmf = new MmFile(name, MmFile.Mode.readWriteNew, 4, null);
760        char[] p = cast(char[]) mmf[];
761        p[] = "1234";
762        mmf[3] = '5';
763        assert(mmf[2] == '3');
764        assert(mmf[3] == '5');
765    }
766    {
767        string p = cast(string) std.file.read(name);
768        assert(p[] == "1235");
769    }
770    {
771        scope MmFile mmf = new MmFile(name, MmFile.Mode.readWriteNew, 4, null);
772        char[] p = cast(char[]) mmf[];
773        p[] = "5678";
774        mmf[3] = '5';
775        assert(mmf[2] == '7');
776        assert(mmf[3] == '5');
777        assert(cast(string) mmf[] == "5675");
778    }
779    {
780        string p = cast(string) std.file.read(name);
781        assert(p[] == "5675");
782    }
783    {
784        scope MmFile mmf = new MmFile(name, MmFile.Mode.readWrite, 4, null);
785        char[] p = cast(char[]) mmf[];
786        assert(cast(char[]) mmf[] == "5675");
787        p[] = "9102";
788        mmf[2] = '5';
789        assert(cast(string) mmf[] == "9152");
790    }
791    {
792        string p = cast(string) std.file.read(name);
793        assert(p[] == "9152");
794    }
795    std.file.remove(name);
796    {
797        scope MmFile mmf = new MmFile(name, MmFile.Mode.readWrite, 4, null);
798        char[] p = cast(char[]) mmf[];
799        p[] = "abcd";
800        mmf[2] = '5';
801        assert(cast(string) mmf[] == "ab5d");
802    }
803    {
804        string p = cast(string) std.file.read(name);
805        assert(p[] == "ab5d");
806    }
807    {
808        scope MmFile mmf = new MmFile(name, MmFile.Mode.readCopyOnWrite, 4, null);
809        char[] p = cast(char[]) mmf[];
810        assert(cast(string) mmf[] == "ab5d");
811        p[] = "9102";
812        mmf[2] = '5';
813        assert(cast(string) mmf[] == "9152");
814    }
815    {
816        string p = cast(string) std.file.read(name);
817        assert(p[] == "ab5d");
818    }
819}
820