1// Copyright 2016 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <ddk/io-buffer.h>
6#include <ddk/debug.h>
7#include <ddk/driver.h>
8#include <zircon/assert.h>
9#include <zircon/process.h>
10#include <zircon/syscalls.h>
11#include <limits.h>
12#include <stdio.h>
13#include <stdlib.h>
14#include <string.h>
15
16// Returns true if a buffer with these parameters was allocated using
17// zx_vmo_create_contiguous.  This is primarily important so we know whether we
18// need to call COMMIT on it to get the pages to exist.
19static bool is_allocated_contiguous(size_t size, uint32_t flags) {
20    return (flags & IO_BUFFER_CONTIG) && size > PAGE_SIZE;
21}
22
23static zx_status_t pin_contig_buffer(zx_handle_t bti, zx_handle_t vmo, size_t size,
24                                     zx_paddr_t* phys, zx_handle_t* pmt) {
25    uint32_t options = ZX_BTI_PERM_READ | ZX_BTI_PERM_WRITE;
26    if (size > PAGE_SIZE) {
27        options |= ZX_BTI_CONTIGUOUS;
28    }
29    return zx_bti_pin(bti, options, vmo, 0, ROUNDUP(size, PAGE_SIZE), phys, 1, pmt);
30}
31
32static zx_status_t io_buffer_init_common(io_buffer_t* buffer, zx_handle_t bti_handle,
33                                         zx_handle_t vmo_handle, size_t size,
34                                         zx_off_t offset, uint32_t flags) {
35    zx_vaddr_t virt;
36
37    zx_vm_option_t map_options = ZX_VM_PERM_READ;
38    if (flags & IO_BUFFER_RW) {
39        map_options = ZX_VM_PERM_READ | ZX_VM_PERM_WRITE;
40    }
41
42    zx_status_t status = zx_vmar_map(zx_vmar_root_self(), map_options, 0, vmo_handle, 0, size, &virt);
43    if (status != ZX_OK) {
44        zxlogf(ERROR, "io_buffer: zx_vmar_map failed %d size: %zu\n", status, size);
45        zx_handle_close(vmo_handle);
46        return status;
47    }
48
49    // For contiguous buffers, pre-lookup the physical mapping so
50    // io_buffer_phys() works.  For non-contiguous buffers, io_buffer_physmap()
51    // will need to be called.
52    zx_paddr_t phys = IO_BUFFER_INVALID_PHYS;
53    zx_handle_t pmt_handle = ZX_HANDLE_INVALID;
54    if (flags & IO_BUFFER_CONTIG) {
55        ZX_DEBUG_ASSERT(offset == 0);
56        status = pin_contig_buffer(bti_handle, vmo_handle, size, &phys, &pmt_handle);
57        if (status != ZX_OK) {
58            zxlogf(ERROR, "io_buffer: init pin failed %d size: %zu\n", status, size);
59            zx_vmar_unmap(zx_vmar_root_self(), virt, size);
60            zx_handle_close(vmo_handle);
61            return status;
62        }
63    }
64
65    buffer->bti_handle = bti_handle;
66    buffer->vmo_handle = vmo_handle;
67    buffer->pmt_handle = pmt_handle;
68    buffer->size = size;
69    buffer->offset = offset;
70    buffer->virt = (void *)virt;
71    buffer->phys = phys;
72
73    return ZX_OK;
74}
75
76zx_status_t io_buffer_init_aligned(io_buffer_t* buffer, zx_handle_t bti, size_t size,
77                                   uint32_t alignment_log2, uint32_t flags) {
78    memset(buffer, 0, sizeof(*buffer));
79
80    if (size == 0) {
81        return ZX_ERR_INVALID_ARGS;
82    }
83    if (flags & ~IO_BUFFER_FLAGS_MASK) {
84        return ZX_ERR_INVALID_ARGS;
85    }
86
87    zx_handle_t vmo_handle;
88    zx_status_t status;
89
90    if (is_allocated_contiguous(size, flags)) {
91        status = zx_vmo_create_contiguous(bti, size, alignment_log2, &vmo_handle);
92    } else {
93        // zx_vmo_create doesn't support passing an alignment.
94        if (alignment_log2 != 0)
95            return ZX_ERR_INVALID_ARGS;
96        status = zx_vmo_create(size, 0, &vmo_handle);
97    }
98    if (status != ZX_OK) {
99        zxlogf(ERROR, "io_buffer: zx_vmo_create failed %d\n", status);
100        return status;
101    }
102
103    if (flags & IO_BUFFER_UNCACHED) {
104        status = zx_vmo_set_cache_policy(vmo_handle, ZX_CACHE_POLICY_UNCACHED);
105        if (status != ZX_OK) {
106            zxlogf(ERROR, "io_buffer: zx_vmo_set_cache_policy failed %d\n", status);
107            zx_handle_close(vmo_handle);
108            return status;
109        }
110    }
111
112    return io_buffer_init_common(buffer, bti, vmo_handle, size, 0, flags);
113}
114
115zx_status_t io_buffer_init(io_buffer_t* buffer, zx_handle_t bti, size_t size, uint32_t flags) {
116    // A zero alignment gets interpreted as PAGE_SIZE_SHIFT.
117    return io_buffer_init_aligned(buffer, bti, size, 0, flags);
118}
119
120zx_status_t io_buffer_init_vmo(io_buffer_t* buffer, zx_handle_t bti, zx_handle_t vmo_handle,
121                               zx_off_t offset, uint32_t flags) {
122    memset(buffer, 0, sizeof(*buffer));
123
124    if (flags != IO_BUFFER_RO && flags != IO_BUFFER_RW) {
125        return ZX_ERR_INVALID_ARGS;
126    }
127
128    zx_status_t status = zx_handle_duplicate(vmo_handle, ZX_RIGHT_SAME_RIGHTS, &vmo_handle);
129    if (status != ZX_OK) return status;
130
131    uint64_t size;
132    status = zx_vmo_get_size(vmo_handle, &size);
133    if (status != ZX_OK) {
134        zx_handle_close(vmo_handle);
135        return status;
136    }
137
138    return io_buffer_init_common(buffer, bti, vmo_handle, size, offset, flags);
139}
140
141zx_status_t io_buffer_init_mmio(io_buffer_t* buffer, zx_handle_t vmo_handle, void* virt,
142                                zx_off_t offset, size_t size) {
143    memset(buffer, 0, sizeof(*buffer));
144
145    zx_status_t status = zx_handle_duplicate(vmo_handle, ZX_RIGHT_SAME_RIGHTS, &vmo_handle);
146    if (status != ZX_OK) return status;
147
148    buffer->vmo_handle = vmo_handle;
149    buffer->size = size;
150    buffer->offset = offset;
151    buffer->virt = (void *)virt;
152
153    return ZX_OK;
154}
155
156zx_status_t io_buffer_init_physical(io_buffer_t* buffer, zx_handle_t bti, zx_paddr_t addr,
157                                    size_t size, zx_handle_t resource, uint32_t cache_policy) {
158    memset(buffer, 0, sizeof(*buffer));
159
160    zx_handle_t vmo_handle;
161    zx_status_t status = zx_vmo_create_physical(resource, addr, size, &vmo_handle);
162    if (status != ZX_OK) {
163        zxlogf(ERROR, "io_buffer: zx_vmo_create_physical failed %d\n", status);
164        return status;
165    }
166
167    status = zx_vmo_set_cache_policy(vmo_handle, cache_policy);
168    if (status != ZX_OK) {
169        zxlogf(ERROR, "io_buffer: zx_vmo_set_cache_policy failed %d\n", status);
170        zx_handle_close(vmo_handle);
171        return status;
172    }
173
174    zx_vm_option_t options = ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_MAP_RANGE;
175    zx_vaddr_t virt;
176    status = zx_vmar_map(zx_vmar_root_self(), options, 0, vmo_handle, 0, size, &virt);
177    if (status != ZX_OK) {
178        zxlogf(ERROR, "io_buffer: zx_vmar_map failed %d size: %zu\n", status, size);
179        zx_handle_close(vmo_handle);
180        return status;
181    }
182
183    zx_paddr_t phys;
184    zx_handle_t pmt;
185    status = pin_contig_buffer(bti, vmo_handle, size, &phys, &pmt);
186    if (status != ZX_OK) {
187        zxlogf(ERROR, "io_buffer: init pin failed %d size: %zu\n", status, size);
188        zx_vmar_unmap(zx_vmar_root_self(), virt, size);
189        zx_handle_close(vmo_handle);
190        return status;
191    }
192
193    buffer->bti_handle = bti;
194    buffer->vmo_handle = vmo_handle;
195    buffer->pmt_handle = pmt;
196    buffer->size = size;
197    buffer->offset = 0;
198    buffer->virt = (void *)virt;
199    buffer->phys = phys;
200    buffer->phys_list = NULL;
201    buffer->phys_count = 0;
202    return ZX_OK;
203}
204
205void io_buffer_release(io_buffer_t* buffer) {
206    if (buffer->vmo_handle != ZX_HANDLE_INVALID) {
207        if (buffer->pmt_handle != ZX_HANDLE_INVALID) {
208            zx_status_t status = zx_pmt_unpin(buffer->pmt_handle);
209            ZX_DEBUG_ASSERT(status == ZX_OK);
210            buffer->pmt_handle = ZX_HANDLE_INVALID;
211        }
212
213        zx_vmar_unmap(zx_vmar_root_self(), (uintptr_t)buffer->virt, buffer->size);
214        zx_handle_close(buffer->vmo_handle);
215        buffer->vmo_handle = ZX_HANDLE_INVALID;
216    }
217    if (buffer->phys_list && buffer->pmt_handle != ZX_HANDLE_INVALID) {
218        zx_status_t status = zx_pmt_unpin(buffer->pmt_handle);
219        ZX_DEBUG_ASSERT(status == ZX_OK);
220        buffer->pmt_handle = ZX_HANDLE_INVALID;
221    }
222    free(buffer->phys_list);
223    buffer->phys_list = NULL;
224    buffer->phys = 0;
225    buffer->phys_count = 0;
226}
227
228zx_status_t io_buffer_cache_op(io_buffer_t* buffer, const uint32_t op,
229                               const zx_off_t offset, const size_t size) {
230    if (size > 0) {
231        return zx_vmo_op_range(buffer->vmo_handle, op, buffer->offset + offset, size, NULL, 0);
232    } else {
233        return ZX_OK;
234    }
235}
236
237zx_status_t io_buffer_cache_flush(io_buffer_t* buffer, zx_off_t offset, size_t length) {
238    if (offset + length < offset || offset + length > buffer->size) {
239        return ZX_ERR_OUT_OF_RANGE;
240    }
241    return zx_cache_flush(io_buffer_virt(buffer) + offset, length, ZX_CACHE_FLUSH_DATA);
242}
243
244zx_status_t io_buffer_cache_flush_invalidate(io_buffer_t* buffer, zx_off_t offset, size_t length) {
245    if (offset + length < offset || offset + length > buffer->size) {
246        return ZX_ERR_OUT_OF_RANGE;
247    }
248    return zx_cache_flush(io_buffer_virt(buffer) + offset, length,
249                          ZX_CACHE_FLUSH_DATA | ZX_CACHE_FLUSH_INVALIDATE);
250}
251
252zx_status_t io_buffer_physmap(io_buffer_t* buffer) {
253    if (buffer->phys_count > 0) {
254        return ZX_OK;
255    }
256    if (buffer->size == 0) {
257        return ZX_ERR_INVALID_ARGS;
258    }
259    if (buffer->pmt_handle != ZX_HANDLE_INVALID && buffer->phys == IO_BUFFER_INVALID_PHYS) {
260        return ZX_ERR_BAD_STATE;
261    }
262
263    // zx_bti_pin returns whole pages, so take into account unaligned vmo
264    // offset and length when calculating the amount of pages returned
265    uint64_t page_offset = ROUNDDOWN(buffer->offset, PAGE_SIZE);
266    // The buffer size is the vmo size from offset 0.
267    uint64_t page_length = buffer->size - page_offset;
268    uint64_t pages = ROUNDUP(page_length, PAGE_SIZE) / PAGE_SIZE;
269
270    zx_paddr_t* paddrs = malloc(pages * sizeof(zx_paddr_t));
271    if (paddrs == NULL) {
272        zxlogf(ERROR, "io_buffer: out of memory\n");
273        return ZX_ERR_NO_MEMORY;
274    }
275
276    if (buffer->phys == IO_BUFFER_INVALID_PHYS) {
277        zx_handle_t pmt;
278        zx_status_t status = io_buffer_physmap_range(buffer, page_offset, page_length,
279                                                     pages, paddrs, &pmt);
280        if (status != ZX_OK) {
281            free(paddrs);
282            return status;
283        }
284        buffer->pmt_handle = pmt;
285    } else {
286        // If this is a contiguous io-buffer, just populate the page array
287        // ourselves.
288        for (size_t i = 0; i < pages; ++i) {
289            paddrs[i] = buffer->phys + page_offset + i * PAGE_SIZE;
290        }
291        paddrs[0] += buffer->offset & (PAGE_SIZE - 1);
292    }
293    buffer->phys_list = paddrs;
294    buffer->phys_count = pages;
295    return ZX_OK;
296}
297
298zx_status_t io_buffer_physmap_range(io_buffer_t* buffer, zx_off_t offset,
299                                    size_t length, size_t phys_count,
300                                    zx_paddr_t* physmap, zx_handle_t* pmt) {
301    // TODO(teisenbe): We need to figure out how to integrate lifetime
302    // management of this pin into the io_buffer API...
303    const size_t sub_offset = offset & (PAGE_SIZE - 1);
304    const size_t pin_offset = offset - sub_offset;
305    const size_t pin_length = ROUNDUP(length + sub_offset, PAGE_SIZE);
306
307    if (pin_length / PAGE_SIZE != phys_count) {
308        return ZX_ERR_INVALID_ARGS;
309    }
310
311    uint32_t options = ZX_BTI_PERM_READ | ZX_BTI_PERM_WRITE;
312    zx_status_t status = zx_bti_pin(buffer->bti_handle, options, buffer->vmo_handle,
313                                    pin_offset, pin_length, physmap, phys_count, pmt);
314    if (status != ZX_OK) {
315        return status;
316    }
317    // Account for the initial misalignment if any
318    physmap[0] += sub_offset;
319    return ZX_OK;
320}
321