1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright(c) 2017 Intel Corporation. All rights reserved. 4 */ 5#include <linux/pagemap.h> 6#include <linux/module.h> 7#include <linux/mount.h> 8#include <linux/pseudo_fs.h> 9#include <linux/magic.h> 10#include <linux/pfn_t.h> 11#include <linux/cdev.h> 12#include <linux/slab.h> 13#include <linux/uio.h> 14#include <linux/dax.h> 15#include <linux/fs.h> 16#include <linux/cacheinfo.h> 17#include "dax-private.h" 18 19/** 20 * struct dax_device - anchor object for dax services 21 * @inode: core vfs 22 * @cdev: optional character interface for "device dax" 23 * @private: dax driver private data 24 * @flags: state and boolean properties 25 * @ops: operations for this device 26 * @holder_data: holder of a dax_device: could be filesystem or mapped device 27 * @holder_ops: operations for the inner holder 28 */ 29struct dax_device { 30 struct inode inode; 31 struct cdev cdev; 32 void *private; 33 unsigned long flags; 34 const struct dax_operations *ops; 35 void *holder_data; 36 const struct dax_holder_operations *holder_ops; 37}; 38 39static dev_t dax_devt; 40DEFINE_STATIC_SRCU(dax_srcu); 41static struct vfsmount *dax_mnt; 42static DEFINE_IDA(dax_minor_ida); 43static struct kmem_cache *dax_cache __read_mostly; 44static struct super_block *dax_superblock __read_mostly; 45 46int dax_read_lock(void) 47{ 48 return srcu_read_lock(&dax_srcu); 49} 50EXPORT_SYMBOL_GPL(dax_read_lock); 51 52void dax_read_unlock(int id) 53{ 54 srcu_read_unlock(&dax_srcu, id); 55} 56EXPORT_SYMBOL_GPL(dax_read_unlock); 57 58#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX) 59#include <linux/blkdev.h> 60 61static DEFINE_XARRAY(dax_hosts); 62 63int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk) 64{ 65 return xa_insert(&dax_hosts, (unsigned long)disk, dax_dev, GFP_KERNEL); 66} 67EXPORT_SYMBOL_GPL(dax_add_host); 68 69void dax_remove_host(struct gendisk *disk) 70{ 71 xa_erase(&dax_hosts, (unsigned long)disk); 72} 73EXPORT_SYMBOL_GPL(dax_remove_host); 74 75/** 76 * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax 77 * @bdev: block device to find a dax_device for 78 * @start_off: returns the byte offset into the dax_device that @bdev starts 79 * @holder: filesystem or mapped device inside the dax_device 80 * @ops: operations for the inner holder 81 */ 82struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off, 83 void *holder, const struct dax_holder_operations *ops) 84{ 85 struct dax_device *dax_dev; 86 u64 part_size; 87 int id; 88 89 if (!blk_queue_dax(bdev->bd_disk->queue)) 90 return NULL; 91 92 *start_off = get_start_sect(bdev) * SECTOR_SIZE; 93 part_size = bdev_nr_sectors(bdev) * SECTOR_SIZE; 94 if (*start_off % PAGE_SIZE || part_size % PAGE_SIZE) { 95 pr_info("%pg: error: unaligned partition for dax\n", bdev); 96 return NULL; 97 } 98 99 id = dax_read_lock(); 100 dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk); 101 if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode)) 102 dax_dev = NULL; 103 else if (holder) { 104 if (!cmpxchg(&dax_dev->holder_data, NULL, holder)) 105 dax_dev->holder_ops = ops; 106 else 107 dax_dev = NULL; 108 } 109 dax_read_unlock(id); 110 111 return dax_dev; 112} 113EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev); 114 115void fs_put_dax(struct dax_device *dax_dev, void *holder) 116{ 117 if (dax_dev && holder && 118 cmpxchg(&dax_dev->holder_data, holder, NULL) == holder) 119 dax_dev->holder_ops = NULL; 120 put_dax(dax_dev); 121} 122EXPORT_SYMBOL_GPL(fs_put_dax); 123#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */ 124 125enum dax_device_flags { 126 /* !alive + rcu grace period == no new operations / mappings */ 127 DAXDEV_ALIVE, 128 /* gate whether dax_flush() calls the low level flush routine */ 129 DAXDEV_WRITE_CACHE, 130 /* flag to check if device supports synchronous flush */ 131 DAXDEV_SYNC, 132 /* do not leave the caches dirty after writes */ 133 DAXDEV_NOCACHE, 134 /* handle CPU fetch exceptions during reads */ 135 DAXDEV_NOMC, 136}; 137 138/** 139 * dax_direct_access() - translate a device pgoff to an absolute pfn 140 * @dax_dev: a dax_device instance representing the logical memory range 141 * @pgoff: offset in pages from the start of the device to translate 142 * @nr_pages: number of consecutive pages caller can handle relative to @pfn 143 * @mode: indicator on normal access or recovery write 144 * @kaddr: output parameter that returns a virtual address mapping of pfn 145 * @pfn: output parameter that returns an absolute pfn translation of @pgoff 146 * 147 * Return: negative errno if an error occurs, otherwise the number of 148 * pages accessible at the device relative @pgoff. 149 */ 150long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, 151 enum dax_access_mode mode, void **kaddr, pfn_t *pfn) 152{ 153 long avail; 154 155 if (!dax_dev) 156 return -EOPNOTSUPP; 157 158 if (!dax_alive(dax_dev)) 159 return -ENXIO; 160 161 if (nr_pages < 0) 162 return -EINVAL; 163 164 avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages, 165 mode, kaddr, pfn); 166 if (!avail) 167 return -ERANGE; 168 return min(avail, nr_pages); 169} 170EXPORT_SYMBOL_GPL(dax_direct_access); 171 172size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, 173 size_t bytes, struct iov_iter *i) 174{ 175 if (!dax_alive(dax_dev)) 176 return 0; 177 178 /* 179 * The userspace address for the memory copy has already been validated 180 * via access_ok() in vfs_write, so use the 'no check' version to bypass 181 * the HARDENED_USERCOPY overhead. 182 */ 183 if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags)) 184 return _copy_from_iter_flushcache(addr, bytes, i); 185 return _copy_from_iter(addr, bytes, i); 186} 187 188size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, 189 size_t bytes, struct iov_iter *i) 190{ 191 if (!dax_alive(dax_dev)) 192 return 0; 193 194 /* 195 * The userspace address for the memory copy has already been validated 196 * via access_ok() in vfs_red, so use the 'no check' version to bypass 197 * the HARDENED_USERCOPY overhead. 198 */ 199 if (test_bit(DAXDEV_NOMC, &dax_dev->flags)) 200 return _copy_mc_to_iter(addr, bytes, i); 201 return _copy_to_iter(addr, bytes, i); 202} 203 204int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, 205 size_t nr_pages) 206{ 207 int ret; 208 209 if (!dax_alive(dax_dev)) 210 return -ENXIO; 211 /* 212 * There are no callers that want to zero more than one page as of now. 213 * Once users are there, this check can be removed after the 214 * device mapper code has been updated to split ranges across targets. 215 */ 216 if (nr_pages != 1) 217 return -EIO; 218 219 ret = dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages); 220 return dax_mem2blk_err(ret); 221} 222EXPORT_SYMBOL_GPL(dax_zero_page_range); 223 224size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, 225 void *addr, size_t bytes, struct iov_iter *iter) 226{ 227 if (!dax_dev->ops->recovery_write) 228 return 0; 229 return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter); 230} 231EXPORT_SYMBOL_GPL(dax_recovery_write); 232 233int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, 234 u64 len, int mf_flags) 235{ 236 int rc, id; 237 238 id = dax_read_lock(); 239 if (!dax_alive(dax_dev)) { 240 rc = -ENXIO; 241 goto out; 242 } 243 244 if (!dax_dev->holder_ops) { 245 rc = -EOPNOTSUPP; 246 goto out; 247 } 248 249 rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags); 250out: 251 dax_read_unlock(id); 252 return rc; 253} 254EXPORT_SYMBOL_GPL(dax_holder_notify_failure); 255 256#ifdef CONFIG_ARCH_HAS_PMEM_API 257void arch_wb_cache_pmem(void *addr, size_t size); 258void dax_flush(struct dax_device *dax_dev, void *addr, size_t size) 259{ 260 if (unlikely(!dax_write_cache_enabled(dax_dev))) 261 return; 262 263 arch_wb_cache_pmem(addr, size); 264} 265#else 266void dax_flush(struct dax_device *dax_dev, void *addr, size_t size) 267{ 268} 269#endif 270EXPORT_SYMBOL_GPL(dax_flush); 271 272void dax_write_cache(struct dax_device *dax_dev, bool wc) 273{ 274 if (wc) 275 set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags); 276 else 277 clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags); 278} 279EXPORT_SYMBOL_GPL(dax_write_cache); 280 281bool dax_write_cache_enabled(struct dax_device *dax_dev) 282{ 283 return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags); 284} 285EXPORT_SYMBOL_GPL(dax_write_cache_enabled); 286 287bool dax_synchronous(struct dax_device *dax_dev) 288{ 289 return test_bit(DAXDEV_SYNC, &dax_dev->flags); 290} 291EXPORT_SYMBOL_GPL(dax_synchronous); 292 293void set_dax_synchronous(struct dax_device *dax_dev) 294{ 295 set_bit(DAXDEV_SYNC, &dax_dev->flags); 296} 297EXPORT_SYMBOL_GPL(set_dax_synchronous); 298 299void set_dax_nocache(struct dax_device *dax_dev) 300{ 301 set_bit(DAXDEV_NOCACHE, &dax_dev->flags); 302} 303EXPORT_SYMBOL_GPL(set_dax_nocache); 304 305void set_dax_nomc(struct dax_device *dax_dev) 306{ 307 set_bit(DAXDEV_NOMC, &dax_dev->flags); 308} 309EXPORT_SYMBOL_GPL(set_dax_nomc); 310 311bool dax_alive(struct dax_device *dax_dev) 312{ 313 lockdep_assert_held(&dax_srcu); 314 return test_bit(DAXDEV_ALIVE, &dax_dev->flags); 315} 316EXPORT_SYMBOL_GPL(dax_alive); 317 318/* 319 * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring 320 * that any fault handlers or operations that might have seen 321 * dax_alive(), have completed. Any operations that start after 322 * synchronize_srcu() has run will abort upon seeing !dax_alive(). 323 * 324 * Note, because alloc_dax() returns an ERR_PTR() on error, callers 325 * typically store its result into a local variable in order to check 326 * the result. Therefore, care must be taken to populate the struct 327 * device dax_dev field make sure the dax_dev is not leaked. 328 */ 329void kill_dax(struct dax_device *dax_dev) 330{ 331 if (!dax_dev) 332 return; 333 334 if (dax_dev->holder_data != NULL) 335 dax_holder_notify_failure(dax_dev, 0, U64_MAX, 336 MF_MEM_PRE_REMOVE); 337 338 clear_bit(DAXDEV_ALIVE, &dax_dev->flags); 339 synchronize_srcu(&dax_srcu); 340 341 /* clear holder data */ 342 dax_dev->holder_ops = NULL; 343 dax_dev->holder_data = NULL; 344} 345EXPORT_SYMBOL_GPL(kill_dax); 346 347void run_dax(struct dax_device *dax_dev) 348{ 349 set_bit(DAXDEV_ALIVE, &dax_dev->flags); 350} 351EXPORT_SYMBOL_GPL(run_dax); 352 353static struct inode *dax_alloc_inode(struct super_block *sb) 354{ 355 struct dax_device *dax_dev; 356 struct inode *inode; 357 358 dax_dev = alloc_inode_sb(sb, dax_cache, GFP_KERNEL); 359 if (!dax_dev) 360 return NULL; 361 362 inode = &dax_dev->inode; 363 inode->i_rdev = 0; 364 return inode; 365} 366 367static struct dax_device *to_dax_dev(struct inode *inode) 368{ 369 return container_of(inode, struct dax_device, inode); 370} 371 372static void dax_free_inode(struct inode *inode) 373{ 374 struct dax_device *dax_dev = to_dax_dev(inode); 375 if (inode->i_rdev) 376 ida_free(&dax_minor_ida, iminor(inode)); 377 kmem_cache_free(dax_cache, dax_dev); 378} 379 380static void dax_destroy_inode(struct inode *inode) 381{ 382 struct dax_device *dax_dev = to_dax_dev(inode); 383 WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags), 384 "kill_dax() must be called before final iput()\n"); 385} 386 387static const struct super_operations dax_sops = { 388 .statfs = simple_statfs, 389 .alloc_inode = dax_alloc_inode, 390 .destroy_inode = dax_destroy_inode, 391 .free_inode = dax_free_inode, 392 .drop_inode = generic_delete_inode, 393}; 394 395static int dax_init_fs_context(struct fs_context *fc) 396{ 397 struct pseudo_fs_context *ctx = init_pseudo(fc, DAXFS_MAGIC); 398 if (!ctx) 399 return -ENOMEM; 400 ctx->ops = &dax_sops; 401 return 0; 402} 403 404static struct file_system_type dax_fs_type = { 405 .name = "dax", 406 .init_fs_context = dax_init_fs_context, 407 .kill_sb = kill_anon_super, 408}; 409 410static int dax_test(struct inode *inode, void *data) 411{ 412 dev_t devt = *(dev_t *) data; 413 414 return inode->i_rdev == devt; 415} 416 417static int dax_set(struct inode *inode, void *data) 418{ 419 dev_t devt = *(dev_t *) data; 420 421 inode->i_rdev = devt; 422 return 0; 423} 424 425static struct dax_device *dax_dev_get(dev_t devt) 426{ 427 struct dax_device *dax_dev; 428 struct inode *inode; 429 430 inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31), 431 dax_test, dax_set, &devt); 432 433 if (!inode) 434 return NULL; 435 436 dax_dev = to_dax_dev(inode); 437 if (inode->i_state & I_NEW) { 438 set_bit(DAXDEV_ALIVE, &dax_dev->flags); 439 inode->i_cdev = &dax_dev->cdev; 440 inode->i_mode = S_IFCHR; 441 inode->i_flags = S_DAX; 442 mapping_set_gfp_mask(&inode->i_data, GFP_USER); 443 unlock_new_inode(inode); 444 } 445 446 return dax_dev; 447} 448 449struct dax_device *alloc_dax(void *private, const struct dax_operations *ops) 450{ 451 struct dax_device *dax_dev; 452 dev_t devt; 453 int minor; 454 455 /* 456 * Unavailable on architectures with virtually aliased data caches, 457 * except for device-dax (NULL operations pointer), which does 458 * not use aliased mappings from the kernel. 459 */ 460 if (ops && cpu_dcache_is_aliasing()) 461 return ERR_PTR(-EOPNOTSUPP); 462 463 if (WARN_ON_ONCE(ops && !ops->zero_page_range)) 464 return ERR_PTR(-EINVAL); 465 466 minor = ida_alloc_max(&dax_minor_ida, MINORMASK, GFP_KERNEL); 467 if (minor < 0) 468 return ERR_PTR(-ENOMEM); 469 470 devt = MKDEV(MAJOR(dax_devt), minor); 471 dax_dev = dax_dev_get(devt); 472 if (!dax_dev) 473 goto err_dev; 474 475 dax_dev->ops = ops; 476 dax_dev->private = private; 477 return dax_dev; 478 479 err_dev: 480 ida_free(&dax_minor_ida, minor); 481 return ERR_PTR(-ENOMEM); 482} 483EXPORT_SYMBOL_GPL(alloc_dax); 484 485void put_dax(struct dax_device *dax_dev) 486{ 487 if (!dax_dev) 488 return; 489 iput(&dax_dev->inode); 490} 491EXPORT_SYMBOL_GPL(put_dax); 492 493/** 494 * dax_holder() - obtain the holder of a dax device 495 * @dax_dev: a dax_device instance 496 * 497 * Return: the holder's data which represents the holder if registered, 498 * otherwize NULL. 499 */ 500void *dax_holder(struct dax_device *dax_dev) 501{ 502 return dax_dev->holder_data; 503} 504EXPORT_SYMBOL_GPL(dax_holder); 505 506/** 507 * inode_dax: convert a public inode into its dax_dev 508 * @inode: An inode with i_cdev pointing to a dax_dev 509 * 510 * Note this is not equivalent to to_dax_dev() which is for private 511 * internal use where we know the inode filesystem type == dax_fs_type. 512 */ 513struct dax_device *inode_dax(struct inode *inode) 514{ 515 struct cdev *cdev = inode->i_cdev; 516 517 return container_of(cdev, struct dax_device, cdev); 518} 519EXPORT_SYMBOL_GPL(inode_dax); 520 521struct inode *dax_inode(struct dax_device *dax_dev) 522{ 523 return &dax_dev->inode; 524} 525EXPORT_SYMBOL_GPL(dax_inode); 526 527void *dax_get_private(struct dax_device *dax_dev) 528{ 529 if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags)) 530 return NULL; 531 return dax_dev->private; 532} 533EXPORT_SYMBOL_GPL(dax_get_private); 534 535static void init_once(void *_dax_dev) 536{ 537 struct dax_device *dax_dev = _dax_dev; 538 struct inode *inode = &dax_dev->inode; 539 540 memset(dax_dev, 0, sizeof(*dax_dev)); 541 inode_init_once(inode); 542} 543 544static int dax_fs_init(void) 545{ 546 int rc; 547 548 dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0, 549 SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, 550 init_once); 551 if (!dax_cache) 552 return -ENOMEM; 553 554 dax_mnt = kern_mount(&dax_fs_type); 555 if (IS_ERR(dax_mnt)) { 556 rc = PTR_ERR(dax_mnt); 557 goto err_mount; 558 } 559 dax_superblock = dax_mnt->mnt_sb; 560 561 return 0; 562 563 err_mount: 564 kmem_cache_destroy(dax_cache); 565 566 return rc; 567} 568 569static void dax_fs_exit(void) 570{ 571 kern_unmount(dax_mnt); 572 rcu_barrier(); 573 kmem_cache_destroy(dax_cache); 574} 575 576static int __init dax_core_init(void) 577{ 578 int rc; 579 580 rc = dax_fs_init(); 581 if (rc) 582 return rc; 583 584 rc = alloc_chrdev_region(&dax_devt, 0, MINORMASK+1, "dax"); 585 if (rc) 586 goto err_chrdev; 587 588 rc = dax_bus_init(); 589 if (rc) 590 goto err_bus; 591 return 0; 592 593err_bus: 594 unregister_chrdev_region(dax_devt, MINORMASK+1); 595err_chrdev: 596 dax_fs_exit(); 597 return 0; 598} 599 600static void __exit dax_core_exit(void) 601{ 602 dax_bus_exit(); 603 unregister_chrdev_region(dax_devt, MINORMASK+1); 604 ida_destroy(&dax_minor_ida); 605 dax_fs_exit(); 606} 607 608MODULE_AUTHOR("Intel Corporation"); 609MODULE_LICENSE("GPL v2"); 610subsys_initcall(dax_core_init); 611module_exit(dax_core_exit); 612