1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Copyright 2014 IBM Corp. 4 */ 5 6#include <linux/spinlock.h> 7#include <linux/module.h> 8#include <linux/export.h> 9#include <linux/kernel.h> 10#include <linux/bitmap.h> 11#include <linux/sched/signal.h> 12#include <linux/poll.h> 13#include <linux/pid.h> 14#include <linux/fs.h> 15#include <linux/mm.h> 16#include <linux/slab.h> 17#include <linux/sched/mm.h> 18#include <linux/mmu_context.h> 19#include <asm/cputable.h> 20#include <asm/current.h> 21#include <asm/copro.h> 22 23#include "cxl.h" 24#include "trace.h" 25 26#define CXL_NUM_MINORS 256 /* Total to reserve */ 27 28#define CXL_AFU_MINOR_D(afu) (CXL_CARD_MINOR(afu->adapter) + 1 + (3 * afu->slice)) 29#define CXL_AFU_MINOR_M(afu) (CXL_AFU_MINOR_D(afu) + 1) 30#define CXL_AFU_MINOR_S(afu) (CXL_AFU_MINOR_D(afu) + 2) 31#define CXL_AFU_MKDEV_D(afu) MKDEV(MAJOR(cxl_dev), CXL_AFU_MINOR_D(afu)) 32#define CXL_AFU_MKDEV_M(afu) MKDEV(MAJOR(cxl_dev), CXL_AFU_MINOR_M(afu)) 33#define CXL_AFU_MKDEV_S(afu) MKDEV(MAJOR(cxl_dev), CXL_AFU_MINOR_S(afu)) 34 35#define CXL_DEVT_AFU(dev) ((MINOR(dev) % CXL_DEV_MINORS - 1) / 3) 36 37#define CXL_DEVT_IS_CARD(dev) (MINOR(dev) % CXL_DEV_MINORS == 0) 38 39static dev_t cxl_dev; 40 41static int __afu_open(struct inode *inode, struct file *file, bool master) 42{ 43 struct cxl *adapter; 44 struct cxl_afu *afu; 45 struct cxl_context *ctx; 46 int adapter_num = CXL_DEVT_ADAPTER(inode->i_rdev); 47 int slice = CXL_DEVT_AFU(inode->i_rdev); 48 int rc = -ENODEV; 49 50 pr_devel("afu_open afu%i.%i\n", slice, adapter_num); 51 52 if (!(adapter = get_cxl_adapter(adapter_num))) 53 return -ENODEV; 54 55 if (slice > adapter->slices) 56 goto err_put_adapter; 57 58 spin_lock(&adapter->afu_list_lock); 59 if (!(afu = adapter->afu[slice])) { 60 spin_unlock(&adapter->afu_list_lock); 61 goto err_put_adapter; 62 } 63 64 /* 65 * taking a ref to the afu so that it doesn't go away 66 * for rest of the function. This ref is released before 67 * we return. 68 */ 69 cxl_afu_get(afu); 70 spin_unlock(&adapter->afu_list_lock); 71 72 if (!afu->current_mode) 73 goto err_put_afu; 74 75 if (!cxl_ops->link_ok(adapter, afu)) { 76 rc = -EIO; 77 goto err_put_afu; 78 } 79 80 if (!(ctx = cxl_context_alloc())) { 81 rc = -ENOMEM; 82 goto err_put_afu; 83 } 84 85 rc = cxl_context_init(ctx, afu, master); 86 if (rc) 87 goto err_put_afu; 88 89 cxl_context_set_mapping(ctx, inode->i_mapping); 90 91 pr_devel("afu_open pe: %i\n", ctx->pe); 92 file->private_data = ctx; 93 94 /* indicate success */ 95 rc = 0; 96 97err_put_afu: 98 /* release the ref taken earlier */ 99 cxl_afu_put(afu); 100err_put_adapter: 101 put_device(&adapter->dev); 102 return rc; 103} 104 105int afu_open(struct inode *inode, struct file *file) 106{ 107 return __afu_open(inode, file, false); 108} 109 110static int afu_master_open(struct inode *inode, struct file *file) 111{ 112 return __afu_open(inode, file, true); 113} 114 115int afu_release(struct inode *inode, struct file *file) 116{ 117 struct cxl_context *ctx = file->private_data; 118 119 pr_devel("%s: closing cxl file descriptor. pe: %i\n", 120 __func__, ctx->pe); 121 cxl_context_detach(ctx); 122 123 124 /* 125 * Delete the context's mapping pointer, unless it's created by the 126 * kernel API, in which case leave it so it can be freed by reclaim_ctx() 127 */ 128 if (!ctx->kernelapi) { 129 mutex_lock(&ctx->mapping_lock); 130 ctx->mapping = NULL; 131 mutex_unlock(&ctx->mapping_lock); 132 } 133 134 /* 135 * At this this point all bottom halfs have finished and we should be 136 * getting no more IRQs from the hardware for this context. Once it's 137 * removed from the IDR (and RCU synchronised) it's safe to free the 138 * sstp and context. 139 */ 140 cxl_context_free(ctx); 141 142 return 0; 143} 144 145static long afu_ioctl_start_work(struct cxl_context *ctx, 146 struct cxl_ioctl_start_work __user *uwork) 147{ 148 struct cxl_ioctl_start_work work; 149 u64 amr = 0; 150 int rc; 151 152 pr_devel("%s: pe: %i\n", __func__, ctx->pe); 153 154 /* Do this outside the status_mutex to avoid a circular dependency with 155 * the locking in cxl_mmap_fault() */ 156 if (copy_from_user(&work, uwork, sizeof(work))) 157 return -EFAULT; 158 159 mutex_lock(&ctx->status_mutex); 160 if (ctx->status != OPENED) { 161 rc = -EIO; 162 goto out; 163 } 164 165 /* 166 * if any of the reserved fields are set or any of the unused 167 * flags are set it's invalid 168 */ 169 if (work.reserved1 || work.reserved2 || work.reserved3 || 170 work.reserved4 || work.reserved5 || 171 (work.flags & ~CXL_START_WORK_ALL)) { 172 rc = -EINVAL; 173 goto out; 174 } 175 176 if (!(work.flags & CXL_START_WORK_NUM_IRQS)) 177 work.num_interrupts = ctx->afu->pp_irqs; 178 else if ((work.num_interrupts < ctx->afu->pp_irqs) || 179 (work.num_interrupts > ctx->afu->irqs_max)) { 180 rc = -EINVAL; 181 goto out; 182 } 183 184 if ((rc = afu_register_irqs(ctx, work.num_interrupts))) 185 goto out; 186 187 if (work.flags & CXL_START_WORK_AMR) 188 amr = work.amr & mfspr(SPRN_UAMOR); 189 190 if (work.flags & CXL_START_WORK_TID) 191 ctx->assign_tidr = true; 192 193 ctx->mmio_err_ff = !!(work.flags & CXL_START_WORK_ERR_FF); 194 195 /* 196 * Increment the mapped context count for adapter. This also checks 197 * if adapter_context_lock is taken. 198 */ 199 rc = cxl_adapter_context_get(ctx->afu->adapter); 200 if (rc) { 201 afu_release_irqs(ctx, ctx); 202 goto out; 203 } 204 205 /* 206 * We grab the PID here and not in the file open to allow for the case 207 * where a process (master, some daemon, etc) has opened the chardev on 208 * behalf of another process, so the AFU's mm gets bound to the process 209 * that performs this ioctl and not the process that opened the file. 210 * Also we grab the PID of the group leader so that if the task that 211 * has performed the attach operation exits the mm context of the 212 * process is still accessible. 213 */ 214 ctx->pid = get_task_pid(current, PIDTYPE_PID); 215 216 /* acquire a reference to the task's mm */ 217 ctx->mm = get_task_mm(current); 218 219 /* ensure this mm_struct can't be freed */ 220 cxl_context_mm_count_get(ctx); 221 222 if (ctx->mm) { 223 /* decrement the use count from above */ 224 mmput(ctx->mm); 225 /* make TLBIs for this context global */ 226 mm_context_add_copro(ctx->mm); 227 } 228 229 /* 230 * Increment driver use count. Enables global TLBIs for hash 231 * and callbacks to handle the segment table 232 */ 233 cxl_ctx_get(); 234 235 /* 236 * A barrier is needed to make sure all TLBIs are global 237 * before we attach and the context starts being used by the 238 * adapter. 239 * 240 * Needed after mm_context_add_copro() for radix and 241 * cxl_ctx_get() for hash/p8. 242 * 243 * The barrier should really be mb(), since it involves a 244 * device. However, it's only useful when we have local 245 * vs. global TLBIs, i.e SMP=y. So keep smp_mb(). 246 */ 247 smp_mb(); 248 249 trace_cxl_attach(ctx, work.work_element_descriptor, work.num_interrupts, amr); 250 251 if ((rc = cxl_ops->attach_process(ctx, false, work.work_element_descriptor, 252 amr))) { 253 afu_release_irqs(ctx, ctx); 254 cxl_adapter_context_put(ctx->afu->adapter); 255 put_pid(ctx->pid); 256 ctx->pid = NULL; 257 cxl_ctx_put(); 258 cxl_context_mm_count_put(ctx); 259 if (ctx->mm) 260 mm_context_remove_copro(ctx->mm); 261 goto out; 262 } 263 264 rc = 0; 265 if (work.flags & CXL_START_WORK_TID) { 266 work.tid = ctx->tidr; 267 if (copy_to_user(uwork, &work, sizeof(work))) 268 rc = -EFAULT; 269 } 270 271 ctx->status = STARTED; 272 273out: 274 mutex_unlock(&ctx->status_mutex); 275 return rc; 276} 277 278static long afu_ioctl_process_element(struct cxl_context *ctx, 279 int __user *upe) 280{ 281 pr_devel("%s: pe: %i\n", __func__, ctx->pe); 282 283 if (copy_to_user(upe, &ctx->external_pe, sizeof(__u32))) 284 return -EFAULT; 285 286 return 0; 287} 288 289static long afu_ioctl_get_afu_id(struct cxl_context *ctx, 290 struct cxl_afu_id __user *upafuid) 291{ 292 struct cxl_afu_id afuid = { 0 }; 293 294 afuid.card_id = ctx->afu->adapter->adapter_num; 295 afuid.afu_offset = ctx->afu->slice; 296 afuid.afu_mode = ctx->afu->current_mode; 297 298 /* set the flag bit in case the afu is a slave */ 299 if (ctx->afu->current_mode == CXL_MODE_DIRECTED && !ctx->master) 300 afuid.flags |= CXL_AFUID_FLAG_SLAVE; 301 302 if (copy_to_user(upafuid, &afuid, sizeof(afuid))) 303 return -EFAULT; 304 305 return 0; 306} 307 308long afu_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 309{ 310 struct cxl_context *ctx = file->private_data; 311 312 if (ctx->status == CLOSED) 313 return -EIO; 314 315 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) 316 return -EIO; 317 318 pr_devel("afu_ioctl\n"); 319 switch (cmd) { 320 case CXL_IOCTL_START_WORK: 321 return afu_ioctl_start_work(ctx, (struct cxl_ioctl_start_work __user *)arg); 322 case CXL_IOCTL_GET_PROCESS_ELEMENT: 323 return afu_ioctl_process_element(ctx, (__u32 __user *)arg); 324 case CXL_IOCTL_GET_AFU_ID: 325 return afu_ioctl_get_afu_id(ctx, (struct cxl_afu_id __user *) 326 arg); 327 } 328 return -EINVAL; 329} 330 331static long afu_compat_ioctl(struct file *file, unsigned int cmd, 332 unsigned long arg) 333{ 334 return afu_ioctl(file, cmd, arg); 335} 336 337int afu_mmap(struct file *file, struct vm_area_struct *vm) 338{ 339 struct cxl_context *ctx = file->private_data; 340 341 /* AFU must be started before we can MMIO */ 342 if (ctx->status != STARTED) 343 return -EIO; 344 345 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) 346 return -EIO; 347 348 return cxl_context_iomap(ctx, vm); 349} 350 351static inline bool ctx_event_pending(struct cxl_context *ctx) 352{ 353 if (ctx->pending_irq || ctx->pending_fault || ctx->pending_afu_err) 354 return true; 355 356 if (ctx->afu_driver_ops && atomic_read(&ctx->afu_driver_events)) 357 return true; 358 359 return false; 360} 361 362__poll_t afu_poll(struct file *file, struct poll_table_struct *poll) 363{ 364 struct cxl_context *ctx = file->private_data; 365 __poll_t mask = 0; 366 unsigned long flags; 367 368 369 poll_wait(file, &ctx->wq, poll); 370 371 pr_devel("afu_poll wait done pe: %i\n", ctx->pe); 372 373 spin_lock_irqsave(&ctx->lock, flags); 374 if (ctx_event_pending(ctx)) 375 mask |= EPOLLIN | EPOLLRDNORM; 376 else if (ctx->status == CLOSED) 377 /* Only error on closed when there are no futher events pending 378 */ 379 mask |= EPOLLERR; 380 spin_unlock_irqrestore(&ctx->lock, flags); 381 382 pr_devel("afu_poll pe: %i returning %#x\n", ctx->pe, mask); 383 384 return mask; 385} 386 387static ssize_t afu_driver_event_copy(struct cxl_context *ctx, 388 char __user *buf, 389 struct cxl_event *event, 390 struct cxl_event_afu_driver_reserved *pl) 391{ 392 /* Check event */ 393 if (!pl) { 394 ctx->afu_driver_ops->event_delivered(ctx, pl, -EINVAL); 395 return -EFAULT; 396 } 397 398 /* Check event size */ 399 event->header.size += pl->data_size; 400 if (event->header.size > CXL_READ_MIN_SIZE) { 401 ctx->afu_driver_ops->event_delivered(ctx, pl, -EINVAL); 402 return -EFAULT; 403 } 404 405 /* Copy event header */ 406 if (copy_to_user(buf, event, sizeof(struct cxl_event_header))) { 407 ctx->afu_driver_ops->event_delivered(ctx, pl, -EFAULT); 408 return -EFAULT; 409 } 410 411 /* Copy event data */ 412 buf += sizeof(struct cxl_event_header); 413 if (copy_to_user(buf, &pl->data, pl->data_size)) { 414 ctx->afu_driver_ops->event_delivered(ctx, pl, -EFAULT); 415 return -EFAULT; 416 } 417 418 ctx->afu_driver_ops->event_delivered(ctx, pl, 0); /* Success */ 419 return event->header.size; 420} 421 422ssize_t afu_read(struct file *file, char __user *buf, size_t count, 423 loff_t *off) 424{ 425 struct cxl_context *ctx = file->private_data; 426 struct cxl_event_afu_driver_reserved *pl = NULL; 427 struct cxl_event event; 428 unsigned long flags; 429 int rc; 430 DEFINE_WAIT(wait); 431 432 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) 433 return -EIO; 434 435 if (count < CXL_READ_MIN_SIZE) 436 return -EINVAL; 437 438 spin_lock_irqsave(&ctx->lock, flags); 439 440 for (;;) { 441 prepare_to_wait(&ctx->wq, &wait, TASK_INTERRUPTIBLE); 442 if (ctx_event_pending(ctx) || (ctx->status == CLOSED)) 443 break; 444 445 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) { 446 rc = -EIO; 447 goto out; 448 } 449 450 if (file->f_flags & O_NONBLOCK) { 451 rc = -EAGAIN; 452 goto out; 453 } 454 455 if (signal_pending(current)) { 456 rc = -ERESTARTSYS; 457 goto out; 458 } 459 460 spin_unlock_irqrestore(&ctx->lock, flags); 461 pr_devel("afu_read going to sleep...\n"); 462 schedule(); 463 pr_devel("afu_read woken up\n"); 464 spin_lock_irqsave(&ctx->lock, flags); 465 } 466 467 finish_wait(&ctx->wq, &wait); 468 469 memset(&event, 0, sizeof(event)); 470 event.header.process_element = ctx->pe; 471 event.header.size = sizeof(struct cxl_event_header); 472 if (ctx->afu_driver_ops && atomic_read(&ctx->afu_driver_events)) { 473 pr_devel("afu_read delivering AFU driver specific event\n"); 474 pl = ctx->afu_driver_ops->fetch_event(ctx); 475 atomic_dec(&ctx->afu_driver_events); 476 event.header.type = CXL_EVENT_AFU_DRIVER; 477 } else if (ctx->pending_irq) { 478 pr_devel("afu_read delivering AFU interrupt\n"); 479 event.header.size += sizeof(struct cxl_event_afu_interrupt); 480 event.header.type = CXL_EVENT_AFU_INTERRUPT; 481 event.irq.irq = find_first_bit(ctx->irq_bitmap, ctx->irq_count) + 1; 482 clear_bit(event.irq.irq - 1, ctx->irq_bitmap); 483 if (bitmap_empty(ctx->irq_bitmap, ctx->irq_count)) 484 ctx->pending_irq = false; 485 } else if (ctx->pending_fault) { 486 pr_devel("afu_read delivering data storage fault\n"); 487 event.header.size += sizeof(struct cxl_event_data_storage); 488 event.header.type = CXL_EVENT_DATA_STORAGE; 489 event.fault.addr = ctx->fault_addr; 490 event.fault.dsisr = ctx->fault_dsisr; 491 ctx->pending_fault = false; 492 } else if (ctx->pending_afu_err) { 493 pr_devel("afu_read delivering afu error\n"); 494 event.header.size += sizeof(struct cxl_event_afu_error); 495 event.header.type = CXL_EVENT_AFU_ERROR; 496 event.afu_error.error = ctx->afu_err; 497 ctx->pending_afu_err = false; 498 } else if (ctx->status == CLOSED) { 499 pr_devel("afu_read fatal error\n"); 500 spin_unlock_irqrestore(&ctx->lock, flags); 501 return -EIO; 502 } else 503 WARN(1, "afu_read must be buggy\n"); 504 505 spin_unlock_irqrestore(&ctx->lock, flags); 506 507 if (event.header.type == CXL_EVENT_AFU_DRIVER) 508 return afu_driver_event_copy(ctx, buf, &event, pl); 509 510 if (copy_to_user(buf, &event, event.header.size)) 511 return -EFAULT; 512 return event.header.size; 513 514out: 515 finish_wait(&ctx->wq, &wait); 516 spin_unlock_irqrestore(&ctx->lock, flags); 517 return rc; 518} 519 520/* 521 * Note: if this is updated, we need to update api.c to patch the new ones in 522 * too 523 */ 524const struct file_operations afu_fops = { 525 .owner = THIS_MODULE, 526 .open = afu_open, 527 .poll = afu_poll, 528 .read = afu_read, 529 .release = afu_release, 530 .unlocked_ioctl = afu_ioctl, 531 .compat_ioctl = afu_compat_ioctl, 532 .mmap = afu_mmap, 533}; 534 535static const struct file_operations afu_master_fops = { 536 .owner = THIS_MODULE, 537 .open = afu_master_open, 538 .poll = afu_poll, 539 .read = afu_read, 540 .release = afu_release, 541 .unlocked_ioctl = afu_ioctl, 542 .compat_ioctl = afu_compat_ioctl, 543 .mmap = afu_mmap, 544}; 545 546 547static char *cxl_devnode(const struct device *dev, umode_t *mode) 548{ 549 if (cpu_has_feature(CPU_FTR_HVMODE) && 550 CXL_DEVT_IS_CARD(dev->devt)) { 551 /* 552 * These minor numbers will eventually be used to program the 553 * PSL and AFUs once we have dynamic reprogramming support 554 */ 555 return NULL; 556 } 557 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 558} 559 560static const struct class cxl_class = { 561 .name = "cxl", 562 .devnode = cxl_devnode, 563}; 564 565static int cxl_add_chardev(struct cxl_afu *afu, dev_t devt, struct cdev *cdev, 566 struct device **chardev, char *postfix, char *desc, 567 const struct file_operations *fops) 568{ 569 struct device *dev; 570 int rc; 571 572 cdev_init(cdev, fops); 573 rc = cdev_add(cdev, devt, 1); 574 if (rc) { 575 dev_err(&afu->dev, "Unable to add %s chardev: %i\n", desc, rc); 576 return rc; 577 } 578 579 dev = device_create(&cxl_class, &afu->dev, devt, afu, 580 "afu%i.%i%s", afu->adapter->adapter_num, afu->slice, postfix); 581 if (IS_ERR(dev)) { 582 rc = PTR_ERR(dev); 583 dev_err(&afu->dev, "Unable to create %s chardev in sysfs: %i\n", desc, rc); 584 goto err; 585 } 586 587 *chardev = dev; 588 589 return 0; 590err: 591 cdev_del(cdev); 592 return rc; 593} 594 595int cxl_chardev_d_afu_add(struct cxl_afu *afu) 596{ 597 return cxl_add_chardev(afu, CXL_AFU_MKDEV_D(afu), &afu->afu_cdev_d, 598 &afu->chardev_d, "d", "dedicated", 599 &afu_master_fops); /* Uses master fops */ 600} 601 602int cxl_chardev_m_afu_add(struct cxl_afu *afu) 603{ 604 return cxl_add_chardev(afu, CXL_AFU_MKDEV_M(afu), &afu->afu_cdev_m, 605 &afu->chardev_m, "m", "master", 606 &afu_master_fops); 607} 608 609int cxl_chardev_s_afu_add(struct cxl_afu *afu) 610{ 611 return cxl_add_chardev(afu, CXL_AFU_MKDEV_S(afu), &afu->afu_cdev_s, 612 &afu->chardev_s, "s", "shared", 613 &afu_fops); 614} 615 616void cxl_chardev_afu_remove(struct cxl_afu *afu) 617{ 618 if (afu->chardev_d) { 619 cdev_del(&afu->afu_cdev_d); 620 device_unregister(afu->chardev_d); 621 afu->chardev_d = NULL; 622 } 623 if (afu->chardev_m) { 624 cdev_del(&afu->afu_cdev_m); 625 device_unregister(afu->chardev_m); 626 afu->chardev_m = NULL; 627 } 628 if (afu->chardev_s) { 629 cdev_del(&afu->afu_cdev_s); 630 device_unregister(afu->chardev_s); 631 afu->chardev_s = NULL; 632 } 633} 634 635int cxl_register_afu(struct cxl_afu *afu) 636{ 637 afu->dev.class = &cxl_class; 638 639 return device_register(&afu->dev); 640} 641 642int cxl_register_adapter(struct cxl *adapter) 643{ 644 adapter->dev.class = &cxl_class; 645 646 /* 647 * Future: When we support dynamically reprogramming the PSL & AFU we 648 * will expose the interface to do that via a chardev: 649 * adapter->dev.devt = CXL_CARD_MKDEV(adapter); 650 */ 651 652 return device_register(&adapter->dev); 653} 654 655dev_t cxl_get_dev(void) 656{ 657 return cxl_dev; 658} 659 660int __init cxl_file_init(void) 661{ 662 int rc; 663 664 /* 665 * If these change we really need to update API. Either change some 666 * flags or update API version number CXL_API_VERSION. 667 */ 668 BUILD_BUG_ON(CXL_API_VERSION != 3); 669 BUILD_BUG_ON(sizeof(struct cxl_ioctl_start_work) != 64); 670 BUILD_BUG_ON(sizeof(struct cxl_event_header) != 8); 671 BUILD_BUG_ON(sizeof(struct cxl_event_afu_interrupt) != 8); 672 BUILD_BUG_ON(sizeof(struct cxl_event_data_storage) != 32); 673 BUILD_BUG_ON(sizeof(struct cxl_event_afu_error) != 16); 674 675 if ((rc = alloc_chrdev_region(&cxl_dev, 0, CXL_NUM_MINORS, "cxl"))) { 676 pr_err("Unable to allocate CXL major number: %i\n", rc); 677 return rc; 678 } 679 680 pr_devel("CXL device allocated, MAJOR %i\n", MAJOR(cxl_dev)); 681 682 rc = class_register(&cxl_class); 683 if (rc) { 684 pr_err("Unable to create CXL class\n"); 685 goto err; 686 } 687 688 return 0; 689 690err: 691 unregister_chrdev_region(cxl_dev, CXL_NUM_MINORS); 692 return rc; 693} 694 695void cxl_file_exit(void) 696{ 697 unregister_chrdev_region(cxl_dev, CXL_NUM_MINORS); 698 class_unregister(&cxl_class); 699} 700