1// Copyright 2017 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <algorithm>
6#include <thread>
7#include <vector>
8
9#include <blobfs/fsck.h>
10#include <fbl/auto_call.h>
11#include <sys/stat.h>
12
13#include "blobfs.h"
14
15namespace {
16
17// Add the blob described by |info| on host to the |blobfs| blobfs store.
18zx_status_t AddBlob(blobfs::Blobfs* blobfs, MerkleInfo& info) {
19    const char* path = info.path.c_str();
20    fbl::unique_fd data_fd(open(path, O_RDONLY, 0644));
21    if (!data_fd) {
22        fprintf(stderr, "error: cannot open '%s'\n", path);
23        return ZX_ERR_IO;
24    }
25    zx_status_t status = blobfs::blobfs_add_blob_with_merkle(blobfs, data_fd.get(), info.length,
26                                                             fbl::move(info.digest),
27                                                             fbl::move(info.merkle));
28    if (status != ZX_OK && status != ZX_ERR_ALREADY_EXISTS) {
29        fprintf(stderr, "blobfs: Failed to add blob '%s': %d\n", path, status);
30        return status;
31    }
32    return ZX_OK;
33}
34
35} // namespace
36
37zx_status_t BlobfsCreator::Usage() {
38    zx_status_t status = FsCreator::Usage();
39
40    // Additional information about manifest format.
41    fprintf(stderr, "\nEach manifest line must adhere to one of the following formats:\n");
42    fprintf(stderr, "\t'dst/path=src/path'\n");
43    fprintf(stderr, "\t'dst/path'\n");
44    fprintf(stderr, "with one dst/src pair or single dst per line.\n");
45    return status;
46}
47
48bool BlobfsCreator::IsCommandValid(Command command) {
49    switch (command) {
50    case Command::kMkfs:
51    case Command::kFsck:
52    case Command::kAdd:
53        return true;
54    default:
55        return false;
56    }
57}
58
59bool BlobfsCreator::IsOptionValid(Option option) {
60    //TODO(planders): Add offset and length support to blobfs.
61    switch (option) {
62    case Option::kDepfile:
63    case Option::kReadonly:
64    case Option::kHelp:
65        return true;
66    default:
67        return false;
68    }
69}
70
71bool BlobfsCreator::IsArgumentValid(Argument argument) {
72    switch (argument) {
73    case Argument::kManifest:
74    case Argument::kBlob:
75        return true;
76    default:
77        return false;
78    }
79}
80
81zx_status_t BlobfsCreator::ProcessManifestLine(FILE* manifest, const char* dir_path) {
82    char src[PATH_MAX];
83    src[0] = '\0';
84    char dst[PATH_MAX];
85    dst[0] = '\0';
86
87    zx_status_t status;
88    if ((status = ParseManifestLine(manifest, dir_path, &src[0], &dst[0])) != ZX_OK) {
89        return status;
90    }
91
92    if (!strlen(src)) {
93        fprintf(stderr, "Manifest line must specify source file\n");
94        return ZX_ERR_INVALID_ARGS;
95    }
96
97    blob_list_.push_back(src);
98    return ZX_OK;
99}
100
101zx_status_t BlobfsCreator::ProcessCustom(int argc, char** argv, uint8_t* processed) {
102    constexpr uint8_t required_args = 2;
103    if (strcmp(argv[0], "--blob")) {
104        fprintf(stderr, "Argument not found: %s\n", argv[0]);
105        return ZX_ERR_INVALID_ARGS;
106    } else if (argc < required_args) {
107        fprintf(stderr, "Not enough arguments for %s\n", argv[0]);
108        return ZX_ERR_INVALID_ARGS;
109    }
110
111    blob_list_.push_back(argv[1]);
112    *processed = required_args;
113    return ZX_OK;
114}
115
116zx_status_t BlobfsCreator::CalculateRequiredSize(off_t* out) {
117    std::vector<std::thread> threads;
118    unsigned blob_index = 0;
119    unsigned n_threads = std::thread::hardware_concurrency();
120    if (!n_threads) {
121        n_threads = 4;
122    }
123    zx_status_t status = ZX_OK;
124    std::mutex mtx;
125    for (unsigned j = n_threads; j > 0; j--) {
126        threads.push_back(std::thread([&] {
127            unsigned i = 0;
128            while (true) {
129                mtx.lock();
130                if (status != ZX_OK) {
131                    mtx.unlock();
132                    return;
133                }
134                i = blob_index++;
135                mtx.unlock();
136                if (i >= blob_list_.size()) {
137                    return;
138                }
139                const char* path = blob_list_[i].c_str();
140                zx_status_t res;
141                if ((res = AppendDepfile(path)) != ZX_OK) {
142                    mtx.lock();
143                    status = res;
144                    mtx.unlock();
145                    return;
146                }
147
148                MerkleInfo info;
149                fbl::unique_fd data_fd(open(path, O_RDONLY, 0644));
150                if ((res = blobfs::blobfs_create_merkle(data_fd.get(), &info.digest,
151                                                        &info.merkle)) != ZX_OK) {
152                    mtx.lock();
153                    status = res;
154                    mtx.unlock();
155                    return;
156                }
157
158                struct stat s;
159                if (fstat(data_fd.get(), &s) < 0) {
160                    mtx.lock();
161                    status = ZX_ERR_BAD_STATE;
162                    mtx.unlock();
163                    return;
164                }
165                info.path = path;
166                info.length = s.st_size;
167
168                mtx.lock();
169                merkle_list_.push_back(fbl::move(info));
170                mtx.unlock();
171            }
172        }));
173    }
174
175    for (unsigned i = 0; i < threads.size(); i++) {
176        threads[i].join();
177    }
178
179    if (status != ZX_OK) {
180        return status;
181    }
182
183    // Remove all duplicate blobs by first sorting the merkle trees by
184    // digest, and then by reshuffling the vector to exclude duplicates.
185    std::sort(merkle_list_.begin(), merkle_list_.end(), DigestCompare());
186    auto compare = [](const MerkleInfo& lhs, const MerkleInfo& rhs) {
187        return lhs.digest == rhs.digest;
188    };
189    auto it = std::unique(merkle_list_.begin(), merkle_list_.end(), compare);
190    merkle_list_.resize(std::distance(merkle_list_.begin(), it));
191
192    for (const auto& info : merkle_list_) {
193        blobfs::blobfs_inode_t node;
194        node.blob_size = info.length;
195        data_blocks_ += MerkleTreeBlocks(node) + BlobDataBlocks(node);
196    }
197
198    blobfs::blobfs_info_t info;
199    info.inode_count = blobfs::kBlobfsDefaultInodeCount;
200    info.block_count = data_blocks_;
201    *out = (data_blocks_ + blobfs::DataStartBlock(info)) * blobfs::kBlobfsBlockSize;
202    return ZX_OK;
203}
204
205zx_status_t BlobfsCreator::Mkfs() {
206    uint64_t block_count;
207    if (blobfs::blobfs_get_blockcount(fd_.get(), &block_count)) {
208        fprintf(stderr, "blobfs: cannot find end of underlying device\n");
209        return ZX_ERR_IO;
210    }
211
212    int r = blobfs::blobfs_mkfs(fd_.get(), block_count);
213
214    if (r >= 0 && !blob_list_.is_empty()) {
215        zx_status_t status;
216        if ((status = Add()) != ZX_OK) {
217            return status;
218        }
219    }
220    return r;
221}
222
223zx_status_t BlobfsCreator::Fsck() {
224    zx_status_t status;
225    fbl::unique_ptr<blobfs::Blobfs> vn;
226    if ((status = blobfs::blobfs_create(&vn, fbl::move(fd_))) < 0) {
227        return status;
228    }
229
230    return blobfs::blobfs_check(fbl::move(vn));
231}
232
233zx_status_t BlobfsCreator::Add() {
234    if (blob_list_.is_empty()) {
235        fprintf(stderr, "Adding a blob requires an additional file argument\n");
236        return Usage();
237    }
238
239    zx_status_t status = ZX_OK;
240    fbl::unique_ptr<blobfs::Blobfs> blobfs;
241    if ((status = blobfs_create(&blobfs, fbl::move(fd_))) != ZX_OK) {
242        return status;
243    }
244
245    std::vector<std::thread> threads;
246    std::mutex mtx;
247
248    unsigned n_threads = std::thread::hardware_concurrency();
249    if (!n_threads) {
250        n_threads = 4;
251    }
252    unsigned blob_index = 0;
253    for (unsigned j = n_threads; j > 0; j--) {
254        threads.push_back(std::thread([&] {
255            unsigned i = 0;
256            while (true) {
257                mtx.lock();
258                i = blob_index++;
259                if (i >= merkle_list_.size()) {
260                    mtx.unlock();
261                    return;
262                }
263                mtx.unlock();
264
265                zx_status_t res;
266                if ((res = AddBlob(blobfs.get(), merkle_list_[i])) < 0) {
267                    mtx.lock();
268                    status = res;
269                    mtx.unlock();
270                    return;
271                }
272            }
273        }));
274    }
275
276    for (unsigned i = 0; i < threads.size(); i++) {
277        threads[i].join();
278    }
279
280    return status;
281}
282
283int main(int argc, char** argv) {
284    BlobfsCreator blobfs;
285
286    if (blobfs.ProcessAndRun(argc, argv) != ZX_OK) {
287        return -1;
288    }
289
290    return 0;
291}
292