1// Written in the D programming language.
2
3/**
4 * Read and write memory mapped files.
5 * Copyright: Copyright Digital Mars 2004 - 2009.
6 * License:   $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
7 * Authors:   $(HTTP digitalmars.com, Walter Bright),
8 *            Matthew Wilson
9 * Source:    $(PHOBOSSRC std/_mmfile.d)
10 *
11 * $(SCRIPT inhibitQuickIndex = 1;)
12 */
13/*          Copyright Digital Mars 2004 - 2009.
14 * Distributed under the Boost Software License, Version 1.0.
15 *    (See accompanying file LICENSE_1_0.txt or copy at
16 *          http://www.boost.org/LICENSE_1_0.txt)
17 */
18module std.mmfile;
19
20import core.stdc.errno;
21import core.stdc.stdio;
22import core.stdc.stdlib;
23import std.conv, std.exception, std.stdio;
24import std.file;
25import std.path;
26import std.string;
27
28import std.internal.cstring;
29
30//debug = MMFILE;
31
32version (Windows)
33{
34    import core.sys.windows.windows;
35    import std.utf;
36    import std.windows.syserror;
37}
38else version (Posix)
39{
40    import core.sys.posix.fcntl;
41    import core.sys.posix.sys.mman;
42    import core.sys.posix.sys.stat;
43    import core.sys.posix.unistd;
44}
45else
46{
47    static assert(0);
48}
49
50/**
51 * MmFile objects control the memory mapped file resource.
52 */
53class MmFile
54{
55    /**
56     * The mode the memory mapped file is opened with.
57     */
58    enum Mode
59    {
60        read,            /// Read existing file
61        readWriteNew,    /// Delete existing file, write new file
62        readWrite,       /// Read/Write existing file, create if not existing
63        readCopyOnWrite, /// Read/Write existing file, copy on write
64    }
65
66    /**
67     * Open memory mapped file filename for reading.
68     * File is closed when the object instance is deleted.
69     * Throws:
70     *  std.file.FileException
71     */
72    this(string filename)
73    {
74        this(filename, Mode.read, 0, null);
75    }
76
77    version (linux) this(File file, Mode mode = Mode.read, ulong size = 0,
78            void* address = null, size_t window = 0)
79    {
80        // Save a copy of the File to make sure the fd stays open.
81        this.file = file;
82        this(file.fileno, mode, size, address, window);
83    }
84
85    version (linux) private this(int fildes, Mode mode, ulong size,
86            void* address, size_t window)
87    {
88        int oflag;
89        int fmode;
90
91        switch (mode)
92        {
93        case Mode.read:
94            flags = MAP_SHARED;
95            prot = PROT_READ;
96            oflag = O_RDONLY;
97            fmode = 0;
98            break;
99
100        case Mode.readWriteNew:
101            assert(size != 0);
102            flags = MAP_SHARED;
103            prot = PROT_READ | PROT_WRITE;
104            oflag = O_CREAT | O_RDWR | O_TRUNC;
105            fmode = octal!660;
106            break;
107
108        case Mode.readWrite:
109            flags = MAP_SHARED;
110            prot = PROT_READ | PROT_WRITE;
111            oflag = O_CREAT | O_RDWR;
112            fmode = octal!660;
113            break;
114
115        case Mode.readCopyOnWrite:
116            flags = MAP_PRIVATE;
117            prot = PROT_READ | PROT_WRITE;
118            oflag = O_RDWR;
119            fmode = 0;
120            break;
121
122        default:
123            assert(0);
124        }
125
126        fd = fildes;
127
128        // Adjust size
129        stat_t statbuf = void;
130        errnoEnforce(fstat(fd, &statbuf) == 0);
131        if (prot & PROT_WRITE && size > statbuf.st_size)
132        {
133            // Need to make the file size bytes big
134            lseek(fd, cast(off_t)(size - 1), SEEK_SET);
135            char c = 0;
136            core.sys.posix.unistd.write(fd, &c, 1);
137        }
138        else if (prot & PROT_READ && size == 0)
139            size = statbuf.st_size;
140        this.size = size;
141
142        // Map the file into memory!
143        size_t initial_map = (window && 2*window<size)
144            ? 2*window : cast(size_t) size;
145        auto p = mmap(address, initial_map, prot, flags, fd, 0);
146        if (p == MAP_FAILED)
147        {
148            errnoEnforce(false, "Could not map file into memory");
149        }
150        data = p[0 .. initial_map];
151    }
152
153    /**
154     * Open memory mapped file filename in mode.
155     * File is closed when the object instance is deleted.
156     * Params:
157     *  filename = name of the file.
158     *      If null, an anonymous file mapping is created.
159     *  mode = access mode defined above.
160     *  size =  the size of the file. If 0, it is taken to be the
161     *      size of the existing file.
162     *  address = the preferred address to map the file to,
163     *      although the system is not required to honor it.
164     *      If null, the system selects the most convenient address.
165     *  window = preferred block size of the amount of data to map at one time
166     *      with 0 meaning map the entire file. The window size must be a
167     *      multiple of the memory allocation page size.
168     * Throws:
169     *  std.file.FileException
170     */
171    this(string filename, Mode mode, ulong size, void* address,
172            size_t window = 0)
173    {
174        this.filename = filename;
175        this.mMode = mode;
176        this.window = window;
177        this.address = address;
178
179        version (Windows)
180        {
181            void* p;
182            uint dwDesiredAccess2;
183            uint dwShareMode;
184            uint dwCreationDisposition;
185            uint flProtect;
186
187            switch (mode)
188            {
189            case Mode.read:
190                dwDesiredAccess2 = GENERIC_READ;
191                dwShareMode = FILE_SHARE_READ;
192                dwCreationDisposition = OPEN_EXISTING;
193                flProtect = PAGE_READONLY;
194                dwDesiredAccess = FILE_MAP_READ;
195                break;
196
197            case Mode.readWriteNew:
198                assert(size != 0);
199                dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
200                dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
201                dwCreationDisposition = CREATE_ALWAYS;
202                flProtect = PAGE_READWRITE;
203                dwDesiredAccess = FILE_MAP_WRITE;
204                break;
205
206            case Mode.readWrite:
207                dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
208                dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
209                dwCreationDisposition = OPEN_ALWAYS;
210                flProtect = PAGE_READWRITE;
211                dwDesiredAccess = FILE_MAP_WRITE;
212                break;
213
214            case Mode.readCopyOnWrite:
215                dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
216                dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
217                dwCreationDisposition = OPEN_EXISTING;
218                flProtect = PAGE_WRITECOPY;
219                dwDesiredAccess = FILE_MAP_COPY;
220                break;
221
222            default:
223                assert(0);
224            }
225
226            if (filename != null)
227            {
228                hFile = CreateFileW(filename.tempCStringW(),
229                        dwDesiredAccess2,
230                        dwShareMode,
231                        null,
232                        dwCreationDisposition,
233                        FILE_ATTRIBUTE_NORMAL,
234                        cast(HANDLE) null);
235                wenforce(hFile != INVALID_HANDLE_VALUE, "CreateFileW");
236            }
237            else
238                hFile = INVALID_HANDLE_VALUE;
239
240            scope(failure)
241            {
242                if (hFile != INVALID_HANDLE_VALUE)
243                {
244                    CloseHandle(hFile);
245                    hFile = INVALID_HANDLE_VALUE;
246                }
247            }
248
249            int hi = cast(int)(size >> 32);
250            hFileMap = CreateFileMappingW(hFile, null, flProtect,
251                    hi, cast(uint) size, null);
252            wenforce(hFileMap, "CreateFileMapping");
253            scope(failure)
254            {
255                CloseHandle(hFileMap);
256                hFileMap = null;
257            }
258
259            if (size == 0 && filename != null)
260            {
261                uint sizehi;
262                uint sizelow = GetFileSize(hFile, &sizehi);
263                wenforce(sizelow != INVALID_FILE_SIZE || GetLastError() != ERROR_SUCCESS,
264                    "GetFileSize");
265                size = (cast(ulong) sizehi << 32) + sizelow;
266            }
267            this.size = size;
268
269            size_t initial_map = (window && 2*window<size)
270                ? 2*window : cast(size_t) size;
271            p = MapViewOfFileEx(hFileMap, dwDesiredAccess, 0, 0,
272                    initial_map, address);
273            wenforce(p, "MapViewOfFileEx");
274            data = p[0 .. initial_map];
275
276            debug (MMFILE) printf("MmFile.this(): p = %p, size = %d\n", p, size);
277        }
278        else version (Posix)
279        {
280            void* p;
281            int oflag;
282            int fmode;
283
284            switch (mode)
285            {
286            case Mode.read:
287                flags = MAP_SHARED;
288                prot = PROT_READ;
289                oflag = O_RDONLY;
290                fmode = 0;
291                break;
292
293            case Mode.readWriteNew:
294                assert(size != 0);
295                flags = MAP_SHARED;
296                prot = PROT_READ | PROT_WRITE;
297                oflag = O_CREAT | O_RDWR | O_TRUNC;
298                fmode = octal!660;
299                break;
300
301            case Mode.readWrite:
302                flags = MAP_SHARED;
303                prot = PROT_READ | PROT_WRITE;
304                oflag = O_CREAT | O_RDWR;
305                fmode = octal!660;
306                break;
307
308            case Mode.readCopyOnWrite:
309                flags = MAP_PRIVATE;
310                prot = PROT_READ | PROT_WRITE;
311                oflag = O_RDWR;
312                fmode = 0;
313                break;
314
315            default:
316                assert(0);
317            }
318
319            if (filename.length)
320            {
321                fd = .open(filename.tempCString(), oflag, fmode);
322                errnoEnforce(fd != -1, "Could not open file "~filename);
323
324                stat_t statbuf;
325                if (fstat(fd, &statbuf))
326                {
327                    //printf("\tfstat error, errno = %d\n", errno);
328                    .close(fd);
329                    fd = -1;
330                    errnoEnforce(false, "Could not stat file "~filename);
331                }
332
333                if (prot & PROT_WRITE && size > statbuf.st_size)
334                {
335                    // Need to make the file size bytes big
336                    .lseek(fd, cast(off_t)(size - 1), SEEK_SET);
337                    char c = 0;
338                    core.sys.posix.unistd.write(fd, &c, 1);
339                }
340                else if (prot & PROT_READ && size == 0)
341                    size = statbuf.st_size;
342            }
343            else
344            {
345                fd = -1;
346                version (CRuntime_Glibc) import core.sys.linux.sys.mman : MAP_ANON;
347                flags |= MAP_ANON;
348            }
349            this.size = size;
350            size_t initial_map = (window && 2*window<size)
351                ? 2*window : cast(size_t) size;
352            p = mmap(address, initial_map, prot, flags, fd, 0);
353            if (p == MAP_FAILED)
354            {
355                if (fd != -1)
356                {
357                    .close(fd);
358                    fd = -1;
359                }
360                errnoEnforce(false, "Could not map file "~filename);
361            }
362
363            data = p[0 .. initial_map];
364        }
365        else
366        {
367            static assert(0);
368        }
369    }
370
371    /**
372     * Flushes pending output and closes the memory mapped file.
373     */
374    ~this()
375    {
376        debug (MMFILE) printf("MmFile.~this()\n");
377        unmap();
378        data = null;
379        version (Windows)
380        {
381            wenforce(hFileMap == null || CloseHandle(hFileMap) == TRUE,
382                    "Could not close file handle");
383            hFileMap = null;
384
385            wenforce(!hFile || hFile == INVALID_HANDLE_VALUE
386                    || CloseHandle(hFile) == TRUE,
387                    "Could not close handle");
388            hFile = INVALID_HANDLE_VALUE;
389        }
390        else version (Posix)
391        {
392            version (linux)
393            {
394                if (file !is File.init)
395                {
396                    // The File destructor will close the file,
397                    // if it is the only remaining reference.
398                    return;
399                }
400            }
401            errnoEnforce(fd == -1 || fd <= 2
402                    || .close(fd) != -1,
403                    "Could not close handle");
404            fd = -1;
405        }
406        else
407        {
408            static assert(0);
409        }
410    }
411
412    /* Flush any pending output.
413     */
414    void flush()
415    {
416        debug (MMFILE) printf("MmFile.flush()\n");
417        version (Windows)
418        {
419            FlushViewOfFile(data.ptr, data.length);
420        }
421        else version (Posix)
422        {
423            int i;
424            i = msync(cast(void*) data, data.length, MS_SYNC);   // sys/mman.h
425            errnoEnforce(i == 0, "msync failed");
426        }
427        else
428        {
429            static assert(0);
430        }
431    }
432
433    /**
434     * Gives size in bytes of the memory mapped file.
435     */
436    @property ulong length() const
437    {
438        debug (MMFILE) printf("MmFile.length()\n");
439        return size;
440    }
441
442    /**
443     * Read-only property returning the file mode.
444     */
445    Mode mode()
446    {
447        debug (MMFILE) printf("MmFile.mode()\n");
448        return mMode;
449    }
450
451    /**
452     * Returns entire file contents as an array.
453     */
454    void[] opSlice()
455    {
456        debug (MMFILE) printf("MmFile.opSlice()\n");
457        return opSlice(0,size);
458    }
459
460    /**
461     * Returns slice of file contents as an array.
462     */
463    void[] opSlice(ulong i1, ulong i2)
464    {
465        debug (MMFILE) printf("MmFile.opSlice(%lld, %lld)\n", i1, i2);
466        ensureMapped(i1,i2);
467        size_t off1 = cast(size_t)(i1-start);
468        size_t off2 = cast(size_t)(i2-start);
469        return data[off1 .. off2];
470    }
471
472    /**
473     * Returns byte at index i in file.
474     */
475    ubyte opIndex(ulong i)
476    {
477        debug (MMFILE) printf("MmFile.opIndex(%lld)\n", i);
478        ensureMapped(i);
479        size_t off = cast(size_t)(i-start);
480        return (cast(ubyte[]) data)[off];
481    }
482
483    /**
484     * Sets and returns byte at index i in file to value.
485     */
486    ubyte opIndexAssign(ubyte value, ulong i)
487    {
488        debug (MMFILE) printf("MmFile.opIndex(%lld, %d)\n", i, value);
489        ensureMapped(i);
490        size_t off = cast(size_t)(i-start);
491        return (cast(ubyte[]) data)[off] = value;
492    }
493
494
495    // return true if the given position is currently mapped
496    private int mapped(ulong i)
497    {
498        debug (MMFILE) printf("MmFile.mapped(%lld, %lld, %d)\n", i,start,
499                data.length);
500        return i >= start && i < start+data.length;
501    }
502
503    // unmap the current range
504    private void unmap()
505    {
506        debug (MMFILE) printf("MmFile.unmap()\n");
507        version (Windows)
508        {
509            wenforce(!data.ptr || UnmapViewOfFile(data.ptr) != FALSE, "UnmapViewOfFile");
510        }
511        else
512        {
513            errnoEnforce(!data.ptr || munmap(cast(void*) data, data.length) == 0,
514                    "munmap failed");
515        }
516        data = null;
517    }
518
519    // map range
520    private void map(ulong start, size_t len)
521    {
522        debug (MMFILE) printf("MmFile.map(%lld, %d)\n", start, len);
523        void* p;
524        if (start+len > size)
525            len = cast(size_t)(size-start);
526        version (Windows)
527        {
528            uint hi = cast(uint)(start >> 32);
529            p = MapViewOfFileEx(hFileMap, dwDesiredAccess, hi, cast(uint) start, len, address);
530            wenforce(p, "MapViewOfFileEx");
531        }
532        else
533        {
534            p = mmap(address, len, prot, flags, fd, cast(off_t) start);
535            errnoEnforce(p != MAP_FAILED);
536        }
537        data = p[0 .. len];
538        this.start = start;
539    }
540
541    // ensure a given position is mapped
542    private void ensureMapped(ulong i)
543    {
544        debug (MMFILE) printf("MmFile.ensureMapped(%lld)\n", i);
545        if (!mapped(i))
546        {
547            unmap();
548            if (window == 0)
549            {
550                map(0,cast(size_t) size);
551            }
552            else
553            {
554                ulong block = i/window;
555                if (block == 0)
556                    map(0,2*window);
557                else
558                    map(window*(block-1),3*window);
559            }
560        }
561    }
562
563    // ensure a given range is mapped
564    private void ensureMapped(ulong i, ulong j)
565    {
566        debug (MMFILE) printf("MmFile.ensureMapped(%lld, %lld)\n", i, j);
567        if (!mapped(i) || !mapped(j-1))
568        {
569            unmap();
570            if (window == 0)
571            {
572                map(0,cast(size_t) size);
573            }
574            else
575            {
576                ulong iblock = i/window;
577                ulong jblock = (j-1)/window;
578                if (iblock == 0)
579                {
580                    map(0,cast(size_t)(window*(jblock+2)));
581                }
582                else
583                {
584                    map(window*(iblock-1),cast(size_t)(window*(jblock-iblock+3)));
585                }
586            }
587        }
588    }
589
590private:
591    string filename;
592    void[] data;
593    ulong  start;
594    size_t window;
595    ulong  size;
596    Mode   mMode;
597    void*  address;
598    version (linux) File file;
599
600    version (Windows)
601    {
602        HANDLE hFile = INVALID_HANDLE_VALUE;
603        HANDLE hFileMap = null;
604        uint dwDesiredAccess;
605    }
606    else version (Posix)
607    {
608        int fd;
609        int prot;
610        int flags;
611        int fmode;
612    }
613    else
614    {
615        static assert(0);
616    }
617
618    // Report error, where errno gives the error number
619    // void errNo()
620    // {
621    //     version (Windows)
622    //     {
623    //         throw new FileException(filename, GetLastError());
624    //     }
625    //     else version (linux)
626    //     {
627    //         throw new FileException(filename, errno);
628    //     }
629    //     else
630    //     {
631    //         static assert(0);
632    //     }
633    // }
634}
635
636@system unittest
637{
638    import core.memory : GC;
639    import std.file : deleteme;
640
641    const size_t K = 1024;
642    size_t win = 64*K; // assume the page size is 64K
643    version (Windows)
644    {
645        /+ these aren't defined in core.sys.windows.windows so let's use default
646         SYSTEM_INFO sysinfo;
647         GetSystemInfo(&sysinfo);
648         win = sysinfo.dwAllocationGranularity;
649         +/
650    }
651    else version (linux)
652    {
653        // getpagesize() is not defined in the unix D headers so use the guess
654    }
655    string test_file = std.file.deleteme ~ "-testing.txt";
656    MmFile mf = new MmFile(test_file,MmFile.Mode.readWriteNew,
657            100*K,null,win);
658    ubyte[] str = cast(ubyte[])"1234567890";
659    ubyte[] data = cast(ubyte[]) mf[0 .. 10];
660    data[] = str[];
661    assert( mf[0 .. 10] == str );
662    data = cast(ubyte[]) mf[50 .. 60];
663    data[] = str[];
664    assert( mf[50 .. 60] == str );
665    ubyte[] data2 = cast(ubyte[]) mf[20*K .. 60*K];
666    assert( data2.length == 40*K );
667    assert( data2[$-1] == 0 );
668    mf[100*K-1] = cast(ubyte)'b';
669    data2 = cast(ubyte[]) mf[21*K .. 100*K];
670    assert( data2.length == 79*K );
671    assert( data2[$-1] == 'b' );
672
673    destroy(mf);
674    GC.free(&mf);
675
676    std.file.remove(test_file);
677    // Create anonymous mapping
678    auto test = new MmFile(null, MmFile.Mode.readWriteNew, 1024*1024, null);
679}
680
681version (linux)
682@system unittest // Issue 14868
683{
684    import std.file : deleteme;
685    import std.typecons : scoped;
686
687    // Test retaining ownership of File/fd
688
689    auto fn = std.file.deleteme ~ "-testing.txt";
690    scope(exit) std.file.remove(fn);
691    File(fn, "wb").writeln("Testing!");
692    scoped!MmFile(File(fn));
693
694    // Test that unique ownership of File actually leads to the fd being closed
695
696    auto f = File(fn);
697    auto fd = f.fileno;
698    {
699        auto mf = scoped!MmFile(f);
700        f = File.init;
701    }
702    assert(.close(fd) == -1);
703}
704
705@system unittest // Issue 14994, 14995
706{
707    import std.file : deleteme;
708    import std.typecons : scoped;
709
710    // Zero-length map may or may not be valid on OSX and NetBSD
711    version (OSX)
712        import std.exception : verifyThrown = collectException;
713    version (NetBSD)
714        import std.exception : verifyThrown = collectException;
715    else
716        import std.exception : verifyThrown = assertThrown;
717
718    auto fn = std.file.deleteme ~ "-testing.txt";
719    scope(exit) std.file.remove(fn);
720    verifyThrown(scoped!MmFile(fn, MmFile.Mode.readWrite, 0, null));
721}
722