1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * VAS user space API for its accelerators (Only NX-GZIP is supported now) 4 * Copyright (C) 2019 Haren Myneni, IBM Corp 5 */ 6 7#define pr_fmt(fmt) "vas-api: " fmt 8 9#include <linux/kernel.h> 10#include <linux/device.h> 11#include <linux/cdev.h> 12#include <linux/fs.h> 13#include <linux/slab.h> 14#include <linux/uaccess.h> 15#include <linux/kthread.h> 16#include <linux/sched/signal.h> 17#include <linux/mmu_context.h> 18#include <linux/io.h> 19#include <asm/vas.h> 20#include <uapi/asm/vas-api.h> 21 22/* 23 * The driver creates the device node that can be used as follows: 24 * For NX-GZIP 25 * 26 * fd = open("/dev/crypto/nx-gzip", O_RDWR); 27 * rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr); 28 * paste_addr = mmap(NULL, PAGE_SIZE, prot, MAP_SHARED, fd, 0ULL). 29 * vas_copy(&crb, 0, 1); 30 * vas_paste(paste_addr, 0, 1); 31 * close(fd) or exit process to close window. 32 * 33 * where "vas_copy" and "vas_paste" are defined in copy-paste.h. 34 * copy/paste returns to the user space directly. So refer NX hardware 35 * documentation for exact copy/paste usage and completion / error 36 * conditions. 37 */ 38 39/* 40 * Wrapper object for the nx-gzip device - there is just one instance of 41 * this node for the whole system. 42 */ 43static struct coproc_dev { 44 struct cdev cdev; 45 struct device *device; 46 char *name; 47 dev_t devt; 48 struct class *class; 49 enum vas_cop_type cop_type; 50 const struct vas_user_win_ops *vops; 51} coproc_device; 52 53struct coproc_instance { 54 struct coproc_dev *coproc; 55 struct vas_window *txwin; 56}; 57 58static char *coproc_devnode(const struct device *dev, umode_t *mode) 59{ 60 return kasprintf(GFP_KERNEL, "crypto/%s", dev_name(dev)); 61} 62 63/* 64 * Take reference to pid and mm 65 */ 66int get_vas_user_win_ref(struct vas_user_win_ref *task_ref) 67{ 68 /* 69 * Window opened by a child thread may not be closed when 70 * it exits. So take reference to its pid and release it 71 * when the window is free by parent thread. 72 * Acquire a reference to the task's pid to make sure 73 * pid will not be re-used - needed only for multithread 74 * applications. 75 */ 76 task_ref->pid = get_task_pid(current, PIDTYPE_PID); 77 /* 78 * Acquire a reference to the task's mm. 79 */ 80 task_ref->mm = get_task_mm(current); 81 if (!task_ref->mm) { 82 put_pid(task_ref->pid); 83 pr_err("pid(%d): mm_struct is not found\n", 84 current->pid); 85 return -EPERM; 86 } 87 88 mmgrab(task_ref->mm); 89 mmput(task_ref->mm); 90 /* 91 * Process closes window during exit. In the case of 92 * multithread application, the child thread can open 93 * window and can exit without closing it. So takes tgid 94 * reference until window closed to make sure tgid is not 95 * reused. 96 */ 97 task_ref->tgid = find_get_pid(task_tgid_vnr(current)); 98 99 return 0; 100} 101 102/* 103 * Successful return must release the task reference with 104 * put_task_struct 105 */ 106static bool ref_get_pid_and_task(struct vas_user_win_ref *task_ref, 107 struct task_struct **tskp, struct pid **pidp) 108{ 109 struct task_struct *tsk; 110 struct pid *pid; 111 112 pid = task_ref->pid; 113 tsk = get_pid_task(pid, PIDTYPE_PID); 114 if (!tsk) { 115 pid = task_ref->tgid; 116 tsk = get_pid_task(pid, PIDTYPE_PID); 117 /* 118 * Parent thread (tgid) will be closing window when it 119 * exits. So should not get here. 120 */ 121 if (WARN_ON_ONCE(!tsk)) 122 return false; 123 } 124 125 /* Return if the task is exiting. */ 126 if (tsk->flags & PF_EXITING) { 127 put_task_struct(tsk); 128 return false; 129 } 130 131 *tskp = tsk; 132 *pidp = pid; 133 134 return true; 135} 136 137/* 138 * Update the CSB to indicate a translation error. 139 * 140 * User space will be polling on CSB after the request is issued. 141 * If NX can handle the request without any issues, it updates CSB. 142 * Whereas if NX encounters page fault, the kernel will handle the 143 * fault and update CSB with translation error. 144 * 145 * If we are unable to update the CSB means copy_to_user failed due to 146 * invalid csb_addr, send a signal to the process. 147 */ 148void vas_update_csb(struct coprocessor_request_block *crb, 149 struct vas_user_win_ref *task_ref) 150{ 151 struct coprocessor_status_block csb; 152 struct kernel_siginfo info; 153 struct task_struct *tsk; 154 void __user *csb_addr; 155 struct pid *pid; 156 int rc; 157 158 /* 159 * NX user space windows can not be opened for task->mm=NULL 160 * and faults will not be generated for kernel requests. 161 */ 162 if (WARN_ON_ONCE(!task_ref->mm)) 163 return; 164 165 csb_addr = (void __user *)be64_to_cpu(crb->csb_addr); 166 167 memset(&csb, 0, sizeof(csb)); 168 csb.cc = CSB_CC_FAULT_ADDRESS; 169 csb.ce = CSB_CE_TERMINATION; 170 csb.cs = 0; 171 csb.count = 0; 172 173 /* 174 * NX operates and returns in BE format as defined CRB struct. 175 * So saves fault_storage_addr in BE as NX pastes in FIFO and 176 * expects user space to convert to CPU format. 177 */ 178 csb.address = crb->stamp.nx.fault_storage_addr; 179 csb.flags = 0; 180 181 /* 182 * Process closes send window after all pending NX requests are 183 * completed. In multi-thread applications, a child thread can 184 * open a window and can exit without closing it. May be some 185 * requests are pending or this window can be used by other 186 * threads later. We should handle faults if NX encounters 187 * pages faults on these requests. Update CSB with translation 188 * error and fault address. If csb_addr passed by user space is 189 * invalid, send SEGV signal to pid saved in window. If the 190 * child thread is not running, send the signal to tgid. 191 * Parent thread (tgid) will close this window upon its exit. 192 * 193 * pid and mm references are taken when window is opened by 194 * process (pid). So tgid is used only when child thread opens 195 * a window and exits without closing it. 196 */ 197 198 if (!ref_get_pid_and_task(task_ref, &tsk, &pid)) 199 return; 200 201 kthread_use_mm(task_ref->mm); 202 rc = copy_to_user(csb_addr, &csb, sizeof(csb)); 203 /* 204 * User space polls on csb.flags (first byte). So add barrier 205 * then copy first byte with csb flags update. 206 */ 207 if (!rc) { 208 csb.flags = CSB_V; 209 /* Make sure update to csb.flags is visible now */ 210 smp_mb(); 211 rc = copy_to_user(csb_addr, &csb, sizeof(u8)); 212 } 213 kthread_unuse_mm(task_ref->mm); 214 put_task_struct(tsk); 215 216 /* Success */ 217 if (!rc) 218 return; 219 220 221 pr_debug("Invalid CSB address 0x%p signalling pid(%d)\n", 222 csb_addr, pid_vnr(pid)); 223 224 clear_siginfo(&info); 225 info.si_signo = SIGSEGV; 226 info.si_errno = EFAULT; 227 info.si_code = SEGV_MAPERR; 228 info.si_addr = csb_addr; 229 /* 230 * process will be polling on csb.flags after request is sent to 231 * NX. So generally CSB update should not fail except when an 232 * application passes invalid csb_addr. So an error message will 233 * be displayed and leave it to user space whether to ignore or 234 * handle this signal. 235 */ 236 rcu_read_lock(); 237 rc = kill_pid_info(SIGSEGV, &info, pid); 238 rcu_read_unlock(); 239 240 pr_devel("pid %d kill_proc_info() rc %d\n", pid_vnr(pid), rc); 241} 242 243void vas_dump_crb(struct coprocessor_request_block *crb) 244{ 245 struct data_descriptor_entry *dde; 246 struct nx_fault_stamp *nx; 247 248 dde = &crb->source; 249 pr_devel("SrcDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n", 250 be64_to_cpu(dde->address), be32_to_cpu(dde->length), 251 dde->count, dde->index, dde->flags); 252 253 dde = &crb->target; 254 pr_devel("TgtDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n", 255 be64_to_cpu(dde->address), be32_to_cpu(dde->length), 256 dde->count, dde->index, dde->flags); 257 258 nx = &crb->stamp.nx; 259 pr_devel("NX Stamp: PSWID 0x%x, FSA 0x%llx, flags 0x%x, FS 0x%x\n", 260 be32_to_cpu(nx->pswid), 261 be64_to_cpu(crb->stamp.nx.fault_storage_addr), 262 nx->flags, nx->fault_status); 263} 264 265static int coproc_open(struct inode *inode, struct file *fp) 266{ 267 struct coproc_instance *cp_inst; 268 269 cp_inst = kzalloc(sizeof(*cp_inst), GFP_KERNEL); 270 if (!cp_inst) 271 return -ENOMEM; 272 273 cp_inst->coproc = container_of(inode->i_cdev, struct coproc_dev, 274 cdev); 275 fp->private_data = cp_inst; 276 277 return 0; 278} 279 280static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg) 281{ 282 void __user *uptr = (void __user *)arg; 283 struct vas_tx_win_open_attr uattr; 284 struct coproc_instance *cp_inst; 285 struct vas_window *txwin; 286 int rc; 287 288 cp_inst = fp->private_data; 289 290 /* 291 * One window for file descriptor 292 */ 293 if (cp_inst->txwin) 294 return -EEXIST; 295 296 rc = copy_from_user(&uattr, uptr, sizeof(uattr)); 297 if (rc) { 298 pr_err("copy_from_user() returns %d\n", rc); 299 return -EFAULT; 300 } 301 302 if (uattr.version != 1) { 303 pr_err("Invalid window open API version\n"); 304 return -EINVAL; 305 } 306 307 if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->open_win) { 308 pr_err("VAS API is not registered\n"); 309 return -EACCES; 310 } 311 312 txwin = cp_inst->coproc->vops->open_win(uattr.vas_id, uattr.flags, 313 cp_inst->coproc->cop_type); 314 if (IS_ERR(txwin)) { 315 pr_err_ratelimited("VAS window open failed rc=%ld\n", 316 PTR_ERR(txwin)); 317 return PTR_ERR(txwin); 318 } 319 320 mutex_init(&txwin->task_ref.mmap_mutex); 321 cp_inst->txwin = txwin; 322 323 return 0; 324} 325 326static int coproc_release(struct inode *inode, struct file *fp) 327{ 328 struct coproc_instance *cp_inst = fp->private_data; 329 int rc; 330 331 if (cp_inst->txwin) { 332 if (cp_inst->coproc->vops && 333 cp_inst->coproc->vops->close_win) { 334 rc = cp_inst->coproc->vops->close_win(cp_inst->txwin); 335 if (rc) 336 return rc; 337 } 338 cp_inst->txwin = NULL; 339 } 340 341 kfree(cp_inst); 342 fp->private_data = NULL; 343 344 /* 345 * We don't know here if user has other receive windows 346 * open, so we can't really call clear_thread_tidr(). 347 * So, once the process calls set_thread_tidr(), the 348 * TIDR value sticks around until process exits, resulting 349 * in an extra copy in restore_sprs(). 350 */ 351 352 return 0; 353} 354 355/* 356 * If the executed instruction that caused the fault was a paste, then 357 * clear regs CR0[EQ], advance NIP, and return 0. Else return error code. 358 */ 359static int do_fail_paste(void) 360{ 361 struct pt_regs *regs = current->thread.regs; 362 u32 instword; 363 364 if (WARN_ON_ONCE(!regs)) 365 return -EINVAL; 366 367 if (WARN_ON_ONCE(!user_mode(regs))) 368 return -EINVAL; 369 370 /* 371 * If we couldn't translate the instruction, the driver should 372 * return success without handling the fault, it will be retried 373 * or the instruction fetch will fault. 374 */ 375 if (get_user(instword, (u32 __user *)(regs->nip))) 376 return -EAGAIN; 377 378 /* 379 * Not a paste instruction, driver may fail the fault. 380 */ 381 if ((instword & PPC_INST_PASTE_MASK) != PPC_INST_PASTE) 382 return -ENOENT; 383 384 regs->ccr &= ~0xe0000000; /* Clear CR0[0-2] to fail paste */ 385 regs_add_return_ip(regs, 4); /* Emulate the paste */ 386 387 return 0; 388} 389 390/* 391 * This fault handler is invoked when the core generates page fault on 392 * the paste address. Happens if the kernel closes window in hypervisor 393 * (on pseries) due to lost credit or the paste address is not mapped. 394 */ 395static vm_fault_t vas_mmap_fault(struct vm_fault *vmf) 396{ 397 struct vm_area_struct *vma = vmf->vma; 398 struct file *fp = vma->vm_file; 399 struct coproc_instance *cp_inst = fp->private_data; 400 struct vas_window *txwin; 401 vm_fault_t fault; 402 u64 paste_addr; 403 int ret; 404 405 /* 406 * window is not opened. Shouldn't expect this error. 407 */ 408 if (!cp_inst || !cp_inst->txwin) { 409 pr_err("Unexpected fault on paste address with TX window closed\n"); 410 return VM_FAULT_SIGBUS; 411 } 412 413 txwin = cp_inst->txwin; 414 /* 415 * When the LPAR lost credits due to core removal or during 416 * migration, invalidate the existing mapping for the current 417 * paste addresses and set windows in-active (zap_vma_pages in 418 * reconfig_close_windows()). 419 * New mapping will be done later after migration or new credits 420 * available. So continue to receive faults if the user space 421 * issue NX request. 422 */ 423 if (txwin->task_ref.vma != vmf->vma) { 424 pr_err("No previous mapping with paste address\n"); 425 return VM_FAULT_SIGBUS; 426 } 427 428 mutex_lock(&txwin->task_ref.mmap_mutex); 429 /* 430 * The window may be inactive due to lost credit (Ex: core 431 * removal with DLPAR). If the window is active again when 432 * the credit is available, map the new paste address at the 433 * window virtual address. 434 */ 435 if (txwin->status == VAS_WIN_ACTIVE) { 436 paste_addr = cp_inst->coproc->vops->paste_addr(txwin); 437 if (paste_addr) { 438 fault = vmf_insert_pfn(vma, vma->vm_start, 439 (paste_addr >> PAGE_SHIFT)); 440 mutex_unlock(&txwin->task_ref.mmap_mutex); 441 return fault; 442 } 443 } 444 mutex_unlock(&txwin->task_ref.mmap_mutex); 445 446 /* 447 * Received this fault due to closing the actual window. 448 * It can happen during migration or lost credits. 449 * Since no mapping, return the paste instruction failure 450 * to the user space. 451 */ 452 ret = do_fail_paste(); 453 /* 454 * The user space can retry several times until success (needed 455 * for migration) or should fallback to SW compression or 456 * manage with the existing open windows if available. 457 * Looking at sysfs interface, it can determine whether these 458 * failures are coming during migration or core removal: 459 * nr_used_credits > nr_total_credits when lost credits 460 */ 461 if (!ret || (ret == -EAGAIN)) 462 return VM_FAULT_NOPAGE; 463 464 return VM_FAULT_SIGBUS; 465} 466 467static const struct vm_operations_struct vas_vm_ops = { 468 .fault = vas_mmap_fault, 469}; 470 471static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) 472{ 473 struct coproc_instance *cp_inst = fp->private_data; 474 struct vas_window *txwin; 475 unsigned long pfn; 476 u64 paste_addr; 477 pgprot_t prot; 478 int rc; 479 480 txwin = cp_inst->txwin; 481 482 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 483 pr_debug("size 0x%zx, PAGE_SIZE 0x%zx\n", 484 (vma->vm_end - vma->vm_start), PAGE_SIZE); 485 return -EINVAL; 486 } 487 488 /* Ensure instance has an open send window */ 489 if (!txwin) { 490 pr_err("No send window open?\n"); 491 return -EINVAL; 492 } 493 494 if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->paste_addr) { 495 pr_err("VAS API is not registered\n"); 496 return -EACCES; 497 } 498 499 /* 500 * The initial mmap is done after the window is opened 501 * with ioctl. But before mmap(), this window can be closed in 502 * the hypervisor due to lost credit (core removal on pseries). 503 * So if the window is not active, return mmap() failure with 504 * -EACCES and expects the user space reissue mmap() when it 505 * is active again or open new window when the credit is available. 506 * mmap_mutex protects the paste address mmap() with DLPAR 507 * close/open event and allows mmap() only when the window is 508 * active. 509 */ 510 mutex_lock(&txwin->task_ref.mmap_mutex); 511 if (txwin->status != VAS_WIN_ACTIVE) { 512 pr_err("Window is not active\n"); 513 rc = -EACCES; 514 goto out; 515 } 516 517 paste_addr = cp_inst->coproc->vops->paste_addr(txwin); 518 if (!paste_addr) { 519 pr_err("Window paste address failed\n"); 520 rc = -EINVAL; 521 goto out; 522 } 523 524 pfn = paste_addr >> PAGE_SHIFT; 525 526 /* flags, page_prot from cxl_mmap(), except we want cachable */ 527 vm_flags_set(vma, VM_IO | VM_PFNMAP); 528 vma->vm_page_prot = pgprot_cached(vma->vm_page_prot); 529 530 prot = __pgprot(pgprot_val(vma->vm_page_prot) | _PAGE_DIRTY); 531 532 rc = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, 533 vma->vm_end - vma->vm_start, prot); 534 535 pr_devel("paste addr %llx at %lx, rc %d\n", paste_addr, 536 vma->vm_start, rc); 537 538 txwin->task_ref.vma = vma; 539 vma->vm_ops = &vas_vm_ops; 540 541out: 542 mutex_unlock(&txwin->task_ref.mmap_mutex); 543 return rc; 544} 545 546static long coproc_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) 547{ 548 switch (cmd) { 549 case VAS_TX_WIN_OPEN: 550 return coproc_ioc_tx_win_open(fp, arg); 551 default: 552 return -EINVAL; 553 } 554} 555 556static struct file_operations coproc_fops = { 557 .open = coproc_open, 558 .release = coproc_release, 559 .mmap = coproc_mmap, 560 .unlocked_ioctl = coproc_ioctl, 561}; 562 563/* 564 * Supporting only nx-gzip coprocessor type now, but this API code 565 * extended to other coprocessor types later. 566 */ 567int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type, 568 const char *name, 569 const struct vas_user_win_ops *vops) 570{ 571 int rc = -EINVAL; 572 dev_t devno; 573 574 rc = alloc_chrdev_region(&coproc_device.devt, 1, 1, name); 575 if (rc) { 576 pr_err("Unable to allocate coproc major number: %i\n", rc); 577 return rc; 578 } 579 580 pr_devel("%s device allocated, dev [%i,%i]\n", name, 581 MAJOR(coproc_device.devt), MINOR(coproc_device.devt)); 582 583 coproc_device.class = class_create(name); 584 if (IS_ERR(coproc_device.class)) { 585 rc = PTR_ERR(coproc_device.class); 586 pr_err("Unable to create %s class %d\n", name, rc); 587 goto err_class; 588 } 589 coproc_device.class->devnode = coproc_devnode; 590 coproc_device.cop_type = cop_type; 591 coproc_device.vops = vops; 592 593 coproc_fops.owner = mod; 594 cdev_init(&coproc_device.cdev, &coproc_fops); 595 596 devno = MKDEV(MAJOR(coproc_device.devt), 0); 597 rc = cdev_add(&coproc_device.cdev, devno, 1); 598 if (rc) { 599 pr_err("cdev_add() failed %d\n", rc); 600 goto err_cdev; 601 } 602 603 coproc_device.device = device_create(coproc_device.class, NULL, 604 devno, NULL, name, MINOR(devno)); 605 if (IS_ERR(coproc_device.device)) { 606 rc = PTR_ERR(coproc_device.device); 607 pr_err("Unable to create coproc-%d %d\n", MINOR(devno), rc); 608 goto err; 609 } 610 611 pr_devel("Added dev [%d,%d]\n", MAJOR(devno), MINOR(devno)); 612 613 return 0; 614 615err: 616 cdev_del(&coproc_device.cdev); 617err_cdev: 618 class_destroy(coproc_device.class); 619err_class: 620 unregister_chrdev_region(coproc_device.devt, 1); 621 return rc; 622} 623 624void vas_unregister_coproc_api(void) 625{ 626 dev_t devno; 627 628 cdev_del(&coproc_device.cdev); 629 devno = MKDEV(MAJOR(coproc_device.devt), 0); 630 device_destroy(coproc_device.class, devno); 631 632 class_destroy(coproc_device.class); 633 unregister_chrdev_region(coproc_device.devt, 1); 634} 635