1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2/* Copyright(c) 2014 - 2020 Intel Corporation */ 3 4#include <crypto/algapi.h> 5#include <linux/module.h> 6#include <linux/mutex.h> 7#include <linux/slab.h> 8#include <linux/fs.h> 9#include <linux/bitops.h> 10#include <linux/pci.h> 11#include <linux/cdev.h> 12#include <linux/uaccess.h> 13 14#include "adf_accel_devices.h" 15#include "adf_common_drv.h" 16#include "adf_cfg.h" 17#include "adf_cfg_common.h" 18#include "adf_cfg_user.h" 19 20#define ADF_CFG_MAX_SECTION 512 21#define ADF_CFG_MAX_KEY_VAL 256 22 23#define DEVICE_NAME "qat_adf_ctl" 24 25static DEFINE_MUTEX(adf_ctl_lock); 26static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg); 27 28static const struct file_operations adf_ctl_ops = { 29 .owner = THIS_MODULE, 30 .unlocked_ioctl = adf_ctl_ioctl, 31 .compat_ioctl = compat_ptr_ioctl, 32}; 33 34struct adf_ctl_drv_info { 35 unsigned int major; 36 struct cdev drv_cdev; 37 struct class *drv_class; 38}; 39 40static struct adf_ctl_drv_info adf_ctl_drv; 41 42static void adf_chr_drv_destroy(void) 43{ 44 device_destroy(adf_ctl_drv.drv_class, MKDEV(adf_ctl_drv.major, 0)); 45 cdev_del(&adf_ctl_drv.drv_cdev); 46 class_destroy(adf_ctl_drv.drv_class); 47 unregister_chrdev_region(MKDEV(adf_ctl_drv.major, 0), 1); 48} 49 50static int adf_chr_drv_create(void) 51{ 52 dev_t dev_id; 53 struct device *drv_device; 54 55 if (alloc_chrdev_region(&dev_id, 0, 1, DEVICE_NAME)) { 56 pr_err("QAT: unable to allocate chrdev region\n"); 57 return -EFAULT; 58 } 59 60 adf_ctl_drv.drv_class = class_create(DEVICE_NAME); 61 if (IS_ERR(adf_ctl_drv.drv_class)) { 62 pr_err("QAT: class_create failed for adf_ctl\n"); 63 goto err_chrdev_unreg; 64 } 65 adf_ctl_drv.major = MAJOR(dev_id); 66 cdev_init(&adf_ctl_drv.drv_cdev, &adf_ctl_ops); 67 if (cdev_add(&adf_ctl_drv.drv_cdev, dev_id, 1)) { 68 pr_err("QAT: cdev add failed\n"); 69 goto err_class_destr; 70 } 71 72 drv_device = device_create(adf_ctl_drv.drv_class, NULL, 73 MKDEV(adf_ctl_drv.major, 0), 74 NULL, DEVICE_NAME); 75 if (IS_ERR(drv_device)) { 76 pr_err("QAT: failed to create device\n"); 77 goto err_cdev_del; 78 } 79 return 0; 80err_cdev_del: 81 cdev_del(&adf_ctl_drv.drv_cdev); 82err_class_destr: 83 class_destroy(adf_ctl_drv.drv_class); 84err_chrdev_unreg: 85 unregister_chrdev_region(dev_id, 1); 86 return -EFAULT; 87} 88 89static int adf_ctl_alloc_resources(struct adf_user_cfg_ctl_data **ctl_data, 90 unsigned long arg) 91{ 92 struct adf_user_cfg_ctl_data *cfg_data; 93 94 cfg_data = kzalloc(sizeof(*cfg_data), GFP_KERNEL); 95 if (!cfg_data) 96 return -ENOMEM; 97 98 /* Initialize device id to NO DEVICE as 0 is a valid device id */ 99 cfg_data->device_id = ADF_CFG_NO_DEVICE; 100 101 if (copy_from_user(cfg_data, (void __user *)arg, sizeof(*cfg_data))) { 102 pr_err("QAT: failed to copy from user cfg_data.\n"); 103 kfree(cfg_data); 104 return -EIO; 105 } 106 107 *ctl_data = cfg_data; 108 return 0; 109} 110 111static int adf_add_key_value_data(struct adf_accel_dev *accel_dev, 112 const char *section, 113 const struct adf_user_cfg_key_val *key_val) 114{ 115 if (key_val->type == ADF_HEX) { 116 long *ptr = (long *)key_val->val; 117 long val = *ptr; 118 119 if (adf_cfg_add_key_value_param(accel_dev, section, 120 key_val->key, (void *)val, 121 key_val->type)) { 122 dev_err(&GET_DEV(accel_dev), 123 "failed to add hex keyvalue.\n"); 124 return -EFAULT; 125 } 126 } else { 127 if (adf_cfg_add_key_value_param(accel_dev, section, 128 key_val->key, key_val->val, 129 key_val->type)) { 130 dev_err(&GET_DEV(accel_dev), 131 "failed to add keyvalue.\n"); 132 return -EFAULT; 133 } 134 } 135 return 0; 136} 137 138static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev, 139 struct adf_user_cfg_ctl_data *ctl_data) 140{ 141 struct adf_user_cfg_key_val key_val; 142 struct adf_user_cfg_key_val *params_head; 143 struct adf_user_cfg_section section, *section_head; 144 int i, j; 145 146 section_head = ctl_data->config_section; 147 148 for (i = 0; section_head && i < ADF_CFG_MAX_SECTION; i++) { 149 if (copy_from_user(§ion, (void __user *)section_head, 150 sizeof(*section_head))) { 151 dev_err(&GET_DEV(accel_dev), 152 "failed to copy section info\n"); 153 goto out_err; 154 } 155 156 if (adf_cfg_section_add(accel_dev, section.name)) { 157 dev_err(&GET_DEV(accel_dev), 158 "failed to add section.\n"); 159 goto out_err; 160 } 161 162 params_head = section.params; 163 164 for (j = 0; params_head && j < ADF_CFG_MAX_KEY_VAL; j++) { 165 if (copy_from_user(&key_val, (void __user *)params_head, 166 sizeof(key_val))) { 167 dev_err(&GET_DEV(accel_dev), 168 "Failed to copy keyvalue.\n"); 169 goto out_err; 170 } 171 if (adf_add_key_value_data(accel_dev, section.name, 172 &key_val)) { 173 goto out_err; 174 } 175 params_head = key_val.next; 176 } 177 section_head = section.next; 178 } 179 return 0; 180out_err: 181 adf_cfg_del_all(accel_dev); 182 return -EFAULT; 183} 184 185static int adf_ctl_ioctl_dev_config(struct file *fp, unsigned int cmd, 186 unsigned long arg) 187{ 188 int ret; 189 struct adf_user_cfg_ctl_data *ctl_data; 190 struct adf_accel_dev *accel_dev; 191 192 ret = adf_ctl_alloc_resources(&ctl_data, arg); 193 if (ret) 194 return ret; 195 196 accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); 197 if (!accel_dev) { 198 ret = -EFAULT; 199 goto out; 200 } 201 202 if (adf_dev_started(accel_dev)) { 203 ret = -EFAULT; 204 goto out; 205 } 206 207 if (adf_copy_key_value_data(accel_dev, ctl_data)) { 208 ret = -EFAULT; 209 goto out; 210 } 211 set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); 212out: 213 kfree(ctl_data); 214 return ret; 215} 216 217static int adf_ctl_is_device_in_use(int id) 218{ 219 struct adf_accel_dev *dev; 220 221 list_for_each_entry(dev, adf_devmgr_get_head(), list) { 222 if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) { 223 if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) { 224 dev_info(&GET_DEV(dev), 225 "device qat_dev%d is busy\n", 226 dev->accel_id); 227 return -EBUSY; 228 } 229 } 230 } 231 return 0; 232} 233 234static void adf_ctl_stop_devices(u32 id) 235{ 236 struct adf_accel_dev *accel_dev; 237 238 list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) { 239 if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) { 240 if (!adf_dev_started(accel_dev)) 241 continue; 242 243 /* First stop all VFs */ 244 if (!accel_dev->is_vf) 245 continue; 246 247 adf_dev_down(accel_dev, false); 248 } 249 } 250 251 list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) { 252 if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) { 253 if (!adf_dev_started(accel_dev)) 254 continue; 255 256 adf_dev_down(accel_dev, false); 257 } 258 } 259} 260 261static int adf_ctl_ioctl_dev_stop(struct file *fp, unsigned int cmd, 262 unsigned long arg) 263{ 264 int ret; 265 struct adf_user_cfg_ctl_data *ctl_data; 266 267 ret = adf_ctl_alloc_resources(&ctl_data, arg); 268 if (ret) 269 return ret; 270 271 if (adf_devmgr_verify_id(ctl_data->device_id)) { 272 pr_err("QAT: Device %d not found\n", ctl_data->device_id); 273 ret = -ENODEV; 274 goto out; 275 } 276 277 ret = adf_ctl_is_device_in_use(ctl_data->device_id); 278 if (ret) 279 goto out; 280 281 if (ctl_data->device_id == ADF_CFG_ALL_DEVICES) 282 pr_info("QAT: Stopping all acceleration devices.\n"); 283 else 284 pr_info("QAT: Stopping acceleration device qat_dev%d.\n", 285 ctl_data->device_id); 286 287 adf_ctl_stop_devices(ctl_data->device_id); 288 289out: 290 kfree(ctl_data); 291 return ret; 292} 293 294static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, 295 unsigned long arg) 296{ 297 int ret; 298 struct adf_user_cfg_ctl_data *ctl_data; 299 struct adf_accel_dev *accel_dev; 300 301 ret = adf_ctl_alloc_resources(&ctl_data, arg); 302 if (ret) 303 return ret; 304 305 ret = -ENODEV; 306 accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); 307 if (!accel_dev) 308 goto out; 309 310 dev_info(&GET_DEV(accel_dev), 311 "Starting acceleration device qat_dev%d.\n", 312 ctl_data->device_id); 313 314 ret = adf_dev_up(accel_dev, false); 315 316 if (ret) { 317 dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n", 318 ctl_data->device_id); 319 adf_dev_down(accel_dev, false); 320 } 321out: 322 kfree(ctl_data); 323 return ret; 324} 325 326static int adf_ctl_ioctl_get_num_devices(struct file *fp, unsigned int cmd, 327 unsigned long arg) 328{ 329 u32 num_devices = 0; 330 331 adf_devmgr_get_num_dev(&num_devices); 332 if (copy_to_user((void __user *)arg, &num_devices, sizeof(num_devices))) 333 return -EFAULT; 334 335 return 0; 336} 337 338static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd, 339 unsigned long arg) 340{ 341 struct adf_hw_device_data *hw_data; 342 struct adf_dev_status_info dev_info; 343 struct adf_accel_dev *accel_dev; 344 345 if (copy_from_user(&dev_info, (void __user *)arg, 346 sizeof(struct adf_dev_status_info))) { 347 pr_err("QAT: failed to copy from user.\n"); 348 return -EFAULT; 349 } 350 351 accel_dev = adf_devmgr_get_dev_by_id(dev_info.accel_id); 352 if (!accel_dev) 353 return -ENODEV; 354 355 hw_data = accel_dev->hw_device; 356 dev_info.state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN; 357 dev_info.num_ae = hw_data->get_num_aes(hw_data); 358 dev_info.num_accel = hw_data->get_num_accels(hw_data); 359 dev_info.num_logical_accel = hw_data->num_logical_accel; 360 dev_info.banks_per_accel = hw_data->num_banks 361 / hw_data->num_logical_accel; 362 strscpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name)); 363 dev_info.instance_id = hw_data->instance_id; 364 dev_info.type = hw_data->dev_class->type; 365 dev_info.bus = accel_to_pci_dev(accel_dev)->bus->number; 366 dev_info.dev = PCI_SLOT(accel_to_pci_dev(accel_dev)->devfn); 367 dev_info.fun = PCI_FUNC(accel_to_pci_dev(accel_dev)->devfn); 368 369 if (copy_to_user((void __user *)arg, &dev_info, 370 sizeof(struct adf_dev_status_info))) { 371 dev_err(&GET_DEV(accel_dev), "failed to copy status.\n"); 372 return -EFAULT; 373 } 374 return 0; 375} 376 377static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) 378{ 379 int ret; 380 381 if (mutex_lock_interruptible(&adf_ctl_lock)) 382 return -EFAULT; 383 384 switch (cmd) { 385 case IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS: 386 ret = adf_ctl_ioctl_dev_config(fp, cmd, arg); 387 break; 388 389 case IOCTL_STOP_ACCEL_DEV: 390 ret = adf_ctl_ioctl_dev_stop(fp, cmd, arg); 391 break; 392 393 case IOCTL_START_ACCEL_DEV: 394 ret = adf_ctl_ioctl_dev_start(fp, cmd, arg); 395 break; 396 397 case IOCTL_GET_NUM_DEVICES: 398 ret = adf_ctl_ioctl_get_num_devices(fp, cmd, arg); 399 break; 400 401 case IOCTL_STATUS_ACCEL_DEV: 402 ret = adf_ctl_ioctl_get_status(fp, cmd, arg); 403 break; 404 default: 405 pr_err_ratelimited("QAT: Invalid ioctl %d\n", cmd); 406 ret = -EFAULT; 407 break; 408 } 409 mutex_unlock(&adf_ctl_lock); 410 return ret; 411} 412 413static int __init adf_register_ctl_device_driver(void) 414{ 415 if (adf_chr_drv_create()) 416 goto err_chr_dev; 417 418 if (adf_init_misc_wq()) 419 goto err_misc_wq; 420 421 if (adf_init_aer()) 422 goto err_aer; 423 424 if (adf_init_pf_wq()) 425 goto err_pf_wq; 426 427 if (adf_init_vf_wq()) 428 goto err_vf_wq; 429 430 if (qat_crypto_register()) 431 goto err_crypto_register; 432 433 if (qat_compression_register()) 434 goto err_compression_register; 435 436 return 0; 437 438err_compression_register: 439 qat_crypto_unregister(); 440err_crypto_register: 441 adf_exit_vf_wq(); 442err_vf_wq: 443 adf_exit_pf_wq(); 444err_pf_wq: 445 adf_exit_aer(); 446err_aer: 447 adf_exit_misc_wq(); 448err_misc_wq: 449 adf_chr_drv_destroy(); 450err_chr_dev: 451 mutex_destroy(&adf_ctl_lock); 452 return -EFAULT; 453} 454 455static void __exit adf_unregister_ctl_device_driver(void) 456{ 457 adf_chr_drv_destroy(); 458 adf_exit_misc_wq(); 459 adf_exit_aer(); 460 adf_exit_vf_wq(); 461 adf_exit_pf_wq(); 462 qat_crypto_unregister(); 463 qat_compression_unregister(); 464 adf_clean_vf_map(false); 465 mutex_destroy(&adf_ctl_lock); 466} 467 468module_init(adf_register_ctl_device_driver); 469module_exit(adf_unregister_ctl_device_driver); 470MODULE_LICENSE("Dual BSD/GPL"); 471MODULE_AUTHOR("Intel"); 472MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); 473MODULE_ALIAS_CRYPTO("intel_qat"); 474MODULE_VERSION(ADF_DRV_VERSION); 475MODULE_IMPORT_NS(CRYPTO_INTERNAL); 476