1// Copyright 2017 The Fuchsia Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include <algorithm> 6#include <thread> 7#include <vector> 8 9#include <blobfs/fsck.h> 10#include <fbl/auto_call.h> 11#include <sys/stat.h> 12 13#include "blobfs.h" 14 15namespace { 16 17// Add the blob described by |info| on host to the |blobfs| blobfs store. 18zx_status_t AddBlob(blobfs::Blobfs* blobfs, MerkleInfo& info) { 19 const char* path = info.path.c_str(); 20 fbl::unique_fd data_fd(open(path, O_RDONLY, 0644)); 21 if (!data_fd) { 22 fprintf(stderr, "error: cannot open '%s'\n", path); 23 return ZX_ERR_IO; 24 } 25 zx_status_t status = blobfs::blobfs_add_blob_with_merkle(blobfs, data_fd.get(), info.length, 26 fbl::move(info.digest), 27 fbl::move(info.merkle)); 28 if (status != ZX_OK && status != ZX_ERR_ALREADY_EXISTS) { 29 fprintf(stderr, "blobfs: Failed to add blob '%s': %d\n", path, status); 30 return status; 31 } 32 return ZX_OK; 33} 34 35} // namespace 36 37zx_status_t BlobfsCreator::Usage() { 38 zx_status_t status = FsCreator::Usage(); 39 40 // Additional information about manifest format. 41 fprintf(stderr, "\nEach manifest line must adhere to one of the following formats:\n"); 42 fprintf(stderr, "\t'dst/path=src/path'\n"); 43 fprintf(stderr, "\t'dst/path'\n"); 44 fprintf(stderr, "with one dst/src pair or single dst per line.\n"); 45 return status; 46} 47 48bool BlobfsCreator::IsCommandValid(Command command) { 49 switch (command) { 50 case Command::kMkfs: 51 case Command::kFsck: 52 case Command::kAdd: 53 return true; 54 default: 55 return false; 56 } 57} 58 59bool BlobfsCreator::IsOptionValid(Option option) { 60 //TODO(planders): Add offset and length support to blobfs. 61 switch (option) { 62 case Option::kDepfile: 63 case Option::kReadonly: 64 case Option::kHelp: 65 return true; 66 default: 67 return false; 68 } 69} 70 71bool BlobfsCreator::IsArgumentValid(Argument argument) { 72 switch (argument) { 73 case Argument::kManifest: 74 case Argument::kBlob: 75 return true; 76 default: 77 return false; 78 } 79} 80 81zx_status_t BlobfsCreator::ProcessManifestLine(FILE* manifest, const char* dir_path) { 82 char src[PATH_MAX]; 83 src[0] = '\0'; 84 char dst[PATH_MAX]; 85 dst[0] = '\0'; 86 87 zx_status_t status; 88 if ((status = ParseManifestLine(manifest, dir_path, &src[0], &dst[0])) != ZX_OK) { 89 return status; 90 } 91 92 if (!strlen(src)) { 93 fprintf(stderr, "Manifest line must specify source file\n"); 94 return ZX_ERR_INVALID_ARGS; 95 } 96 97 blob_list_.push_back(src); 98 return ZX_OK; 99} 100 101zx_status_t BlobfsCreator::ProcessCustom(int argc, char** argv, uint8_t* processed) { 102 constexpr uint8_t required_args = 2; 103 if (strcmp(argv[0], "--blob")) { 104 fprintf(stderr, "Argument not found: %s\n", argv[0]); 105 return ZX_ERR_INVALID_ARGS; 106 } else if (argc < required_args) { 107 fprintf(stderr, "Not enough arguments for %s\n", argv[0]); 108 return ZX_ERR_INVALID_ARGS; 109 } 110 111 blob_list_.push_back(argv[1]); 112 *processed = required_args; 113 return ZX_OK; 114} 115 116zx_status_t BlobfsCreator::CalculateRequiredSize(off_t* out) { 117 std::vector<std::thread> threads; 118 unsigned blob_index = 0; 119 unsigned n_threads = std::thread::hardware_concurrency(); 120 if (!n_threads) { 121 n_threads = 4; 122 } 123 zx_status_t status = ZX_OK; 124 std::mutex mtx; 125 for (unsigned j = n_threads; j > 0; j--) { 126 threads.push_back(std::thread([&] { 127 unsigned i = 0; 128 while (true) { 129 mtx.lock(); 130 if (status != ZX_OK) { 131 mtx.unlock(); 132 return; 133 } 134 i = blob_index++; 135 mtx.unlock(); 136 if (i >= blob_list_.size()) { 137 return; 138 } 139 const char* path = blob_list_[i].c_str(); 140 zx_status_t res; 141 if ((res = AppendDepfile(path)) != ZX_OK) { 142 mtx.lock(); 143 status = res; 144 mtx.unlock(); 145 return; 146 } 147 148 MerkleInfo info; 149 fbl::unique_fd data_fd(open(path, O_RDONLY, 0644)); 150 if ((res = blobfs::blobfs_create_merkle(data_fd.get(), &info.digest, 151 &info.merkle)) != ZX_OK) { 152 mtx.lock(); 153 status = res; 154 mtx.unlock(); 155 return; 156 } 157 158 struct stat s; 159 if (fstat(data_fd.get(), &s) < 0) { 160 mtx.lock(); 161 status = ZX_ERR_BAD_STATE; 162 mtx.unlock(); 163 return; 164 } 165 info.path = path; 166 info.length = s.st_size; 167 168 mtx.lock(); 169 merkle_list_.push_back(fbl::move(info)); 170 mtx.unlock(); 171 } 172 })); 173 } 174 175 for (unsigned i = 0; i < threads.size(); i++) { 176 threads[i].join(); 177 } 178 179 if (status != ZX_OK) { 180 return status; 181 } 182 183 // Remove all duplicate blobs by first sorting the merkle trees by 184 // digest, and then by reshuffling the vector to exclude duplicates. 185 std::sort(merkle_list_.begin(), merkle_list_.end(), DigestCompare()); 186 auto compare = [](const MerkleInfo& lhs, const MerkleInfo& rhs) { 187 return lhs.digest == rhs.digest; 188 }; 189 auto it = std::unique(merkle_list_.begin(), merkle_list_.end(), compare); 190 merkle_list_.resize(std::distance(merkle_list_.begin(), it)); 191 192 for (const auto& info : merkle_list_) { 193 blobfs::blobfs_inode_t node; 194 node.blob_size = info.length; 195 data_blocks_ += MerkleTreeBlocks(node) + BlobDataBlocks(node); 196 } 197 198 blobfs::blobfs_info_t info; 199 info.inode_count = blobfs::kBlobfsDefaultInodeCount; 200 info.block_count = data_blocks_; 201 *out = (data_blocks_ + blobfs::DataStartBlock(info)) * blobfs::kBlobfsBlockSize; 202 return ZX_OK; 203} 204 205zx_status_t BlobfsCreator::Mkfs() { 206 uint64_t block_count; 207 if (blobfs::blobfs_get_blockcount(fd_.get(), &block_count)) { 208 fprintf(stderr, "blobfs: cannot find end of underlying device\n"); 209 return ZX_ERR_IO; 210 } 211 212 int r = blobfs::blobfs_mkfs(fd_.get(), block_count); 213 214 if (r >= 0 && !blob_list_.is_empty()) { 215 zx_status_t status; 216 if ((status = Add()) != ZX_OK) { 217 return status; 218 } 219 } 220 return r; 221} 222 223zx_status_t BlobfsCreator::Fsck() { 224 zx_status_t status; 225 fbl::unique_ptr<blobfs::Blobfs> vn; 226 if ((status = blobfs::blobfs_create(&vn, fbl::move(fd_))) < 0) { 227 return status; 228 } 229 230 return blobfs::blobfs_check(fbl::move(vn)); 231} 232 233zx_status_t BlobfsCreator::Add() { 234 if (blob_list_.is_empty()) { 235 fprintf(stderr, "Adding a blob requires an additional file argument\n"); 236 return Usage(); 237 } 238 239 zx_status_t status = ZX_OK; 240 fbl::unique_ptr<blobfs::Blobfs> blobfs; 241 if ((status = blobfs_create(&blobfs, fbl::move(fd_))) != ZX_OK) { 242 return status; 243 } 244 245 std::vector<std::thread> threads; 246 std::mutex mtx; 247 248 unsigned n_threads = std::thread::hardware_concurrency(); 249 if (!n_threads) { 250 n_threads = 4; 251 } 252 unsigned blob_index = 0; 253 for (unsigned j = n_threads; j > 0; j--) { 254 threads.push_back(std::thread([&] { 255 unsigned i = 0; 256 while (true) { 257 mtx.lock(); 258 i = blob_index++; 259 if (i >= merkle_list_.size()) { 260 mtx.unlock(); 261 return; 262 } 263 mtx.unlock(); 264 265 zx_status_t res; 266 if ((res = AddBlob(blobfs.get(), merkle_list_[i])) < 0) { 267 mtx.lock(); 268 status = res; 269 mtx.unlock(); 270 return; 271 } 272 } 273 })); 274 } 275 276 for (unsigned i = 0; i < threads.size(); i++) { 277 threads[i].join(); 278 } 279 280 return status; 281} 282 283int main(int argc, char** argv) { 284 BlobfsCreator blobfs; 285 286 if (blobfs.ProcessAndRun(argc, argv) != ZX_OK) { 287 return -1; 288 } 289 290 return 0; 291} 292