1/* SPDX-License-Identifier: GPL-2.0-only */ 2#ifndef __CARD_BASE_H__ 3#define __CARD_BASE_H__ 4 5/** 6 * IBM Accelerator Family 'GenWQE' 7 * 8 * (C) Copyright IBM Corp. 2013 9 * 10 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com> 11 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com> 12 * Author: Michael Jung <mijung@gmx.net> 13 * Author: Michael Ruettger <michael@ibmra.de> 14 */ 15 16/* 17 * Interfaces within the GenWQE module. Defines genwqe_card and 18 * ddcb_queue as well as ddcb_requ. 19 */ 20 21#include <linux/kernel.h> 22#include <linux/types.h> 23#include <linux/cdev.h> 24#include <linux/stringify.h> 25#include <linux/pci.h> 26#include <linux/semaphore.h> 27#include <linux/uaccess.h> 28#include <linux/io.h> 29#include <linux/debugfs.h> 30#include <linux/slab.h> 31 32#include <linux/genwqe/genwqe_card.h> 33#include "genwqe_driver.h" 34 35#define GENWQE_MSI_IRQS 4 /* Just one supported, no MSIx */ 36 37#define GENWQE_MAX_VFS 15 /* maximum 15 VFs are possible */ 38#define GENWQE_MAX_FUNCS 16 /* 1 PF and 15 VFs */ 39#define GENWQE_CARD_NO_MAX (16 * GENWQE_MAX_FUNCS) 40 41/* Compile parameters, some of them appear in debugfs for later adjustment */ 42#define GENWQE_DDCB_MAX 32 /* DDCBs on the work-queue */ 43#define GENWQE_POLLING_ENABLED 0 /* in case of irqs not working */ 44#define GENWQE_DDCB_SOFTWARE_TIMEOUT 10 /* timeout per DDCB in seconds */ 45#define GENWQE_KILL_TIMEOUT 8 /* time until process gets killed */ 46#define GENWQE_VF_JOBTIMEOUT_MSEC 250 /* 250 msec */ 47#define GENWQE_PF_JOBTIMEOUT_MSEC 8000 /* 8 sec should be ok */ 48#define GENWQE_HEALTH_CHECK_INTERVAL 4 /* <= 0: disabled */ 49 50/* Sysfs attribute groups used when we create the genwqe device */ 51extern const struct attribute_group *genwqe_attribute_groups[]; 52 53/* 54 * Config space for Genwqe5 A7: 55 * 00:[14 10 4b 04]40 00 10 00[00 00 00 12]00 00 00 00 56 * 10: 0c 00 00 f0 07 3c 00 00 00 00 00 00 00 00 00 00 57 * 20: 00 00 00 00 00 00 00 00 00 00 00 00[14 10 4b 04] 58 * 30: 00 00 00 00 50 00 00 00 00 00 00 00 00 00 00 00 59 */ 60#define PCI_DEVICE_GENWQE 0x044b /* Genwqe DeviceID */ 61 62#define PCI_SUBSYSTEM_ID_GENWQE5 0x035f /* Genwqe A5 Subsystem-ID */ 63#define PCI_SUBSYSTEM_ID_GENWQE5_NEW 0x044b /* Genwqe A5 Subsystem-ID */ 64#define PCI_CLASSCODE_GENWQE5 0x1200 /* UNKNOWN */ 65 66#define PCI_SUBVENDOR_ID_IBM_SRIOV 0x0000 67#define PCI_SUBSYSTEM_ID_GENWQE5_SRIOV 0x0000 /* Genwqe A5 Subsystem-ID */ 68#define PCI_CLASSCODE_GENWQE5_SRIOV 0x1200 /* UNKNOWN */ 69 70#define GENWQE_SLU_ARCH_REQ 2 /* Required SLU architecture level */ 71 72/** 73 * struct genwqe_reg - Genwqe data dump functionality 74 */ 75struct genwqe_reg { 76 u32 addr; 77 u32 idx; 78 u64 val; 79}; 80 81/* 82 * enum genwqe_dbg_type - Specify chip unit to dump/debug 83 */ 84enum genwqe_dbg_type { 85 GENWQE_DBG_UNIT0 = 0, /* captured before prev errs cleared */ 86 GENWQE_DBG_UNIT1 = 1, 87 GENWQE_DBG_UNIT2 = 2, 88 GENWQE_DBG_UNIT3 = 3, 89 GENWQE_DBG_UNIT4 = 4, 90 GENWQE_DBG_UNIT5 = 5, 91 GENWQE_DBG_UNIT6 = 6, 92 GENWQE_DBG_UNIT7 = 7, 93 GENWQE_DBG_REGS = 8, 94 GENWQE_DBG_DMA = 9, 95 GENWQE_DBG_UNITS = 10, /* max number of possible debug units */ 96}; 97 98/* Software error injection to simulate card failures */ 99#define GENWQE_INJECT_HARDWARE_FAILURE 0x00000001 /* injects -1 reg reads */ 100#define GENWQE_INJECT_BUS_RESET_FAILURE 0x00000002 /* pci_bus_reset fail */ 101#define GENWQE_INJECT_GFIR_FATAL 0x00000004 /* GFIR = 0x0000ffff */ 102#define GENWQE_INJECT_GFIR_INFO 0x00000008 /* GFIR = 0xffff0000 */ 103 104/* 105 * Genwqe card description and management data. 106 * 107 * Error-handling in case of card malfunction 108 * ------------------------------------------ 109 * 110 * If the card is detected to be defective the outside environment 111 * will cause the PCI layer to call deinit (the cleanup function for 112 * probe). This is the same effect like doing a unbind/bind operation 113 * on the card. 114 * 115 * The genwqe card driver implements a health checking thread which 116 * verifies the card function. If this detects a problem the cards 117 * device is being shutdown and restarted again, along with a reset of 118 * the card and queue. 119 * 120 * All functions accessing the card device return either -EIO or -ENODEV 121 * code to indicate the malfunction to the user. The user has to close 122 * the file descriptor and open a new one, once the card becomes 123 * available again. 124 * 125 * If the open file descriptor is setup to receive SIGIO, the signal is 126 * genereated for the application which has to provide a handler to 127 * react on it. If the application does not close the open 128 * file descriptor a SIGKILL is send to enforce freeing the cards 129 * resources. 130 * 131 * I did not find a different way to prevent kernel problems due to 132 * reference counters for the cards character devices getting out of 133 * sync. The character device deallocation does not block, even if 134 * there is still an open file descriptor pending. If this pending 135 * descriptor is closed, the data structures used by the character 136 * device is reinstantiated, which will lead to the reference counter 137 * dropping below the allowed values. 138 * 139 * Card recovery 140 * ------------- 141 * 142 * To test the internal driver recovery the following command can be used: 143 * sudo sh -c 'echo 0xfffff > /sys/class/genwqe/genwqe0_card/err_inject' 144 */ 145 146 147/** 148 * struct dma_mapping_type - Mapping type definition 149 * 150 * To avoid memcpying data arround we use user memory directly. To do 151 * this we need to pin/swap-in the memory and request a DMA address 152 * for it. 153 */ 154enum dma_mapping_type { 155 GENWQE_MAPPING_RAW = 0, /* contignous memory buffer */ 156 GENWQE_MAPPING_SGL_TEMP, /* sglist dynamically used */ 157 GENWQE_MAPPING_SGL_PINNED, /* sglist used with pinning */ 158}; 159 160/** 161 * struct dma_mapping - Information about memory mappings done by the driver 162 */ 163struct dma_mapping { 164 enum dma_mapping_type type; 165 166 void *u_vaddr; /* user-space vaddr/non-aligned */ 167 void *k_vaddr; /* kernel-space vaddr/non-aligned */ 168 dma_addr_t dma_addr; /* physical DMA address */ 169 170 struct page **page_list; /* list of pages used by user buff */ 171 dma_addr_t *dma_list; /* list of dma addresses per page */ 172 unsigned int nr_pages; /* number of pages */ 173 unsigned int size; /* size in bytes */ 174 175 struct list_head card_list; /* list of usr_maps for card */ 176 struct list_head pin_list; /* list of pinned memory for dev */ 177 int write; /* writable map? useful in unmapping */ 178}; 179 180static inline void genwqe_mapping_init(struct dma_mapping *m, 181 enum dma_mapping_type type) 182{ 183 memset(m, 0, sizeof(*m)); 184 m->type = type; 185 m->write = 1; /* Assume the maps we create are R/W */ 186} 187 188/** 189 * struct ddcb_queue - DDCB queue data 190 * @ddcb_max: Number of DDCBs on the queue 191 * @ddcb_next: Next free DDCB 192 * @ddcb_act: Next DDCB supposed to finish 193 * @ddcb_seq: Sequence number of last DDCB 194 * @ddcbs_in_flight: Currently enqueued DDCBs 195 * @ddcbs_completed: Number of already completed DDCBs 196 * @return_on_busy: Number of -EBUSY returns on full queue 197 * @wait_on_busy: Number of waits on full queue 198 * @ddcb_daddr: DMA address of first DDCB in the queue 199 * @ddcb_vaddr: Kernel virtual address of first DDCB in the queue 200 * @ddcb_req: Associated requests (one per DDCB) 201 * @ddcb_waitqs: Associated wait queues (one per DDCB) 202 * @ddcb_lock: Lock to protect queuing operations 203 * @ddcb_waitq: Wait on next DDCB finishing 204 */ 205 206struct ddcb_queue { 207 int ddcb_max; /* amount of DDCBs */ 208 int ddcb_next; /* next available DDCB num */ 209 int ddcb_act; /* DDCB to be processed */ 210 u16 ddcb_seq; /* slc seq num */ 211 unsigned int ddcbs_in_flight; /* number of ddcbs in processing */ 212 unsigned int ddcbs_completed; 213 unsigned int ddcbs_max_in_flight; 214 unsigned int return_on_busy; /* how many times -EBUSY? */ 215 unsigned int wait_on_busy; 216 217 dma_addr_t ddcb_daddr; /* DMA address */ 218 struct ddcb *ddcb_vaddr; /* kernel virtual addr for DDCBs */ 219 struct ddcb_requ **ddcb_req; /* ddcb processing parameter */ 220 wait_queue_head_t *ddcb_waitqs; /* waitqueue per ddcb */ 221 222 spinlock_t ddcb_lock; /* exclusive access to queue */ 223 wait_queue_head_t busy_waitq; /* wait for ddcb processing */ 224 225 /* registers or the respective queue to be used */ 226 u32 IO_QUEUE_CONFIG; 227 u32 IO_QUEUE_STATUS; 228 u32 IO_QUEUE_SEGMENT; 229 u32 IO_QUEUE_INITSQN; 230 u32 IO_QUEUE_WRAP; 231 u32 IO_QUEUE_OFFSET; 232 u32 IO_QUEUE_WTIME; 233 u32 IO_QUEUE_ERRCNTS; 234 u32 IO_QUEUE_LRW; 235}; 236 237/* 238 * GFIR, SLU_UNITCFG, APP_UNITCFG 239 * 8 Units with FIR/FEC + 64 * 2ndary FIRS/FEC. 240 */ 241#define GENWQE_FFDC_REGS (3 + (8 * (2 + 2 * 64))) 242 243struct genwqe_ffdc { 244 unsigned int entries; 245 struct genwqe_reg *regs; 246}; 247 248/** 249 * struct genwqe_dev - GenWQE device information 250 * @card_state: Card operation state, see above 251 * @ffdc: First Failure Data Capture buffers for each unit 252 * @card_thread: Working thread to operate the DDCB queue 253 * @card_waitq: Wait queue used in card_thread 254 * @queue: DDCB queue 255 * @health_thread: Card monitoring thread (only for PFs) 256 * @health_waitq: Wait queue used in health_thread 257 * @pci_dev: Associated PCI device (function) 258 * @mmio: Base address of 64-bit register space 259 * @mmio_len: Length of register area 260 * @file_lock: Lock to protect access to file_list 261 * @file_list: List of all processes with open GenWQE file descriptors 262 * 263 * This struct contains all information needed to communicate with a 264 * GenWQE card. It is initialized when a GenWQE device is found and 265 * destroyed when it goes away. It holds data to maintain the queue as 266 * well as data needed to feed the user interfaces. 267 */ 268struct genwqe_dev { 269 enum genwqe_card_state card_state; 270 spinlock_t print_lock; 271 272 int card_idx; /* card index 0..CARD_NO_MAX-1 */ 273 u64 flags; /* general flags */ 274 275 /* FFDC data gathering */ 276 struct genwqe_ffdc ffdc[GENWQE_DBG_UNITS]; 277 278 /* DDCB workqueue */ 279 struct task_struct *card_thread; 280 wait_queue_head_t queue_waitq; 281 struct ddcb_queue queue; /* genwqe DDCB queue */ 282 unsigned int irqs_processed; 283 284 /* Card health checking thread */ 285 struct task_struct *health_thread; 286 wait_queue_head_t health_waitq; 287 288 int use_platform_recovery; /* use platform recovery mechanisms */ 289 290 /* char device */ 291 dev_t devnum_genwqe; /* major/minor num card */ 292 const struct class *class_genwqe; /* reference to class object */ 293 struct device *dev; /* for device creation */ 294 struct cdev cdev_genwqe; /* char device for card */ 295 296 struct dentry *debugfs_root; /* debugfs card root directory */ 297 struct dentry *debugfs_genwqe; /* debugfs driver root directory */ 298 299 /* pci resources */ 300 struct pci_dev *pci_dev; /* PCI device */ 301 void __iomem *mmio; /* BAR-0 MMIO start */ 302 unsigned long mmio_len; 303 int num_vfs; 304 u32 vf_jobtimeout_msec[GENWQE_MAX_VFS]; 305 int is_privileged; /* access to all regs possible */ 306 307 /* config regs which we need often */ 308 u64 slu_unitcfg; 309 u64 app_unitcfg; 310 u64 softreset; 311 u64 err_inject; 312 u64 last_gfir; 313 char app_name[5]; 314 315 spinlock_t file_lock; /* lock for open files */ 316 struct list_head file_list; /* list of open files */ 317 318 /* debugfs parameters */ 319 int ddcb_software_timeout; /* wait until DDCB times out */ 320 int skip_recovery; /* circumvention if recovery fails */ 321 int kill_timeout; /* wait after sending SIGKILL */ 322}; 323 324/** 325 * enum genwqe_requ_state - State of a DDCB execution request 326 */ 327enum genwqe_requ_state { 328 GENWQE_REQU_NEW = 0, 329 GENWQE_REQU_ENQUEUED = 1, 330 GENWQE_REQU_TAPPED = 2, 331 GENWQE_REQU_FINISHED = 3, 332 GENWQE_REQU_STATE_MAX, 333}; 334 335/** 336 * struct genwqe_sgl - Scatter gather list describing user-space memory 337 * @sgl: scatter gather list needs to be 128 byte aligned 338 * @sgl_dma_addr: dma address of sgl 339 * @sgl_size: size of area used for sgl 340 * @user_addr: user-space address of memory area 341 * @user_size: size of user-space memory area 342 * @page: buffer for partial pages if needed 343 * @page_dma_addr: dma address partial pages 344 * @write: should we write it back to userspace? 345 */ 346struct genwqe_sgl { 347 dma_addr_t sgl_dma_addr; 348 struct sg_entry *sgl; 349 size_t sgl_size; /* size of sgl */ 350 351 void __user *user_addr; /* user-space base-address */ 352 size_t user_size; /* size of memory area */ 353 354 int write; 355 356 unsigned long nr_pages; 357 unsigned long fpage_offs; 358 size_t fpage_size; 359 size_t lpage_size; 360 361 void *fpage; 362 dma_addr_t fpage_dma_addr; 363 364 void *lpage; 365 dma_addr_t lpage_dma_addr; 366}; 367 368int genwqe_alloc_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl, 369 void __user *user_addr, size_t user_size, int write); 370 371int genwqe_setup_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl, 372 dma_addr_t *dma_list); 373 374int genwqe_free_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl); 375 376/** 377 * struct ddcb_requ - Kernel internal representation of the DDCB request 378 * @cmd: User space representation of the DDCB execution request 379 */ 380struct ddcb_requ { 381 /* kernel specific content */ 382 enum genwqe_requ_state req_state; /* request status */ 383 int num; /* ddcb_no for this request */ 384 struct ddcb_queue *queue; /* associated queue */ 385 386 struct dma_mapping dma_mappings[DDCB_FIXUPS]; 387 struct genwqe_sgl sgls[DDCB_FIXUPS]; 388 389 /* kernel/user shared content */ 390 struct genwqe_ddcb_cmd cmd; /* ddcb_no for this request */ 391 struct genwqe_debug_data debug_data; 392}; 393 394/** 395 * struct genwqe_file - Information for open GenWQE devices 396 */ 397struct genwqe_file { 398 struct genwqe_dev *cd; 399 struct genwqe_driver *client; 400 struct file *filp; 401 402 struct fasync_struct *async_queue; 403 struct pid *opener; 404 struct list_head list; /* entry in list of open files */ 405 406 spinlock_t map_lock; /* lock for dma_mappings */ 407 struct list_head map_list; /* list of dma_mappings */ 408 409 spinlock_t pin_lock; /* lock for pinned memory */ 410 struct list_head pin_list; /* list of pinned memory */ 411}; 412 413int genwqe_setup_service_layer(struct genwqe_dev *cd); /* for PF only */ 414int genwqe_finish_queue(struct genwqe_dev *cd); 415int genwqe_release_service_layer(struct genwqe_dev *cd); 416 417/** 418 * genwqe_get_slu_id() - Read Service Layer Unit Id 419 * Return: 0x00: Development code 420 * 0x01: SLC1 (old) 421 * 0x02: SLC2 (sept2012) 422 * 0x03: SLC2 (feb2013, generic driver) 423 */ 424static inline int genwqe_get_slu_id(struct genwqe_dev *cd) 425{ 426 return (int)((cd->slu_unitcfg >> 32) & 0xff); 427} 428 429int genwqe_ddcbs_in_flight(struct genwqe_dev *cd); 430 431u8 genwqe_card_type(struct genwqe_dev *cd); 432int genwqe_card_reset(struct genwqe_dev *cd); 433int genwqe_set_interrupt_capability(struct genwqe_dev *cd, int count); 434void genwqe_reset_interrupt_capability(struct genwqe_dev *cd); 435 436int genwqe_device_create(struct genwqe_dev *cd); 437int genwqe_device_remove(struct genwqe_dev *cd); 438 439/* debugfs */ 440void genwqe_init_debugfs(struct genwqe_dev *cd); 441void genqwe_exit_debugfs(struct genwqe_dev *cd); 442 443int genwqe_read_softreset(struct genwqe_dev *cd); 444 445/* Hardware Circumventions */ 446int genwqe_recovery_on_fatal_gfir_required(struct genwqe_dev *cd); 447int genwqe_flash_readback_fails(struct genwqe_dev *cd); 448 449/** 450 * genwqe_write_vreg() - Write register in VF window 451 * @cd: genwqe device 452 * @reg: register address 453 * @val: value to write 454 * @func: 0: PF, 1: VF0, ..., 15: VF14 455 */ 456int genwqe_write_vreg(struct genwqe_dev *cd, u32 reg, u64 val, int func); 457 458/** 459 * genwqe_read_vreg() - Read register in VF window 460 * @cd: genwqe device 461 * @reg: register address 462 * @func: 0: PF, 1: VF0, ..., 15: VF14 463 * 464 * Return: content of the register 465 */ 466u64 genwqe_read_vreg(struct genwqe_dev *cd, u32 reg, int func); 467 468/* FFDC Buffer Management */ 469int genwqe_ffdc_buff_size(struct genwqe_dev *cd, int unit_id); 470int genwqe_ffdc_buff_read(struct genwqe_dev *cd, int unit_id, 471 struct genwqe_reg *regs, unsigned int max_regs); 472int genwqe_read_ffdc_regs(struct genwqe_dev *cd, struct genwqe_reg *regs, 473 unsigned int max_regs, int all); 474int genwqe_ffdc_dump_dma(struct genwqe_dev *cd, 475 struct genwqe_reg *regs, unsigned int max_regs); 476 477int genwqe_init_debug_data(struct genwqe_dev *cd, 478 struct genwqe_debug_data *d); 479 480void genwqe_init_crc32(void); 481int genwqe_read_app_id(struct genwqe_dev *cd, char *app_name, int len); 482 483/* Memory allocation/deallocation; dma address handling */ 484int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, 485 void *uaddr, unsigned long size); 486 487int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m); 488 489static inline bool dma_mapping_used(struct dma_mapping *m) 490{ 491 if (!m) 492 return false; 493 return m->size != 0; 494} 495 496/** 497 * __genwqe_execute_ddcb() - Execute DDCB request with addr translation 498 * 499 * This function will do the address translation changes to the DDCBs 500 * according to the definitions required by the ATS field. It looks up 501 * the memory allocation buffer or does vmap/vunmap for the respective 502 * user-space buffers, inclusive page pinning and scatter gather list 503 * buildup and teardown. 504 */ 505int __genwqe_execute_ddcb(struct genwqe_dev *cd, 506 struct genwqe_ddcb_cmd *cmd, unsigned int f_flags); 507 508/** 509 * __genwqe_execute_raw_ddcb() - Execute DDCB request without addr translation 510 * 511 * This version will not do address translation or any modification of 512 * the DDCB data. It is used e.g. for the MoveFlash DDCB which is 513 * entirely prepared by the driver itself. That means the appropriate 514 * DMA addresses are already in the DDCB and do not need any 515 * modification. 516 */ 517int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd, 518 struct genwqe_ddcb_cmd *cmd, 519 unsigned int f_flags); 520int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, 521 struct ddcb_requ *req, 522 unsigned int f_flags); 523 524int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req); 525int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req); 526 527/* register access */ 528int __genwqe_writeq(struct genwqe_dev *cd, u64 byte_offs, u64 val); 529u64 __genwqe_readq(struct genwqe_dev *cd, u64 byte_offs); 530int __genwqe_writel(struct genwqe_dev *cd, u64 byte_offs, u32 val); 531u32 __genwqe_readl(struct genwqe_dev *cd, u64 byte_offs); 532 533void *__genwqe_alloc_consistent(struct genwqe_dev *cd, size_t size, 534 dma_addr_t *dma_handle); 535void __genwqe_free_consistent(struct genwqe_dev *cd, size_t size, 536 void *vaddr, dma_addr_t dma_handle); 537 538/* Base clock frequency in MHz */ 539int genwqe_base_clock_frequency(struct genwqe_dev *cd); 540 541/* Before FFDC is captured the traps should be stopped. */ 542void genwqe_stop_traps(struct genwqe_dev *cd); 543void genwqe_start_traps(struct genwqe_dev *cd); 544 545/* Hardware circumvention */ 546bool genwqe_need_err_masking(struct genwqe_dev *cd); 547 548/** 549 * genwqe_is_privileged() - Determine operation mode for PCI function 550 * 551 * On Intel with SRIOV support we see: 552 * PF: is_physfn = 1 is_virtfn = 0 553 * VF: is_physfn = 0 is_virtfn = 1 554 * 555 * On Systems with no SRIOV support _and_ virtualized systems we get: 556 * is_physfn = 0 is_virtfn = 0 557 * 558 * Other vendors have individual pci device ids to distinguish between 559 * virtual function drivers and physical function drivers. GenWQE 560 * unfortunately has just on pci device id for both, VFs and PF. 561 * 562 * The following code is used to distinguish if the card is running in 563 * privileged mode, either as true PF or in a virtualized system with 564 * full register access e.g. currently on PowerPC. 565 * 566 * if (pci_dev->is_virtfn) 567 * cd->is_privileged = 0; 568 * else 569 * cd->is_privileged = (__genwqe_readq(cd, IO_SLU_BITSTREAM) 570 * != IO_ILLEGAL_VALUE); 571 */ 572static inline int genwqe_is_privileged(struct genwqe_dev *cd) 573{ 574 return cd->is_privileged; 575} 576 577#endif /* __CARD_BASE_H__ */ 578