1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 2023 Advanced Micro Devices, Inc. */ 3 4#include <linux/io.h> 5#include <linux/types.h> 6#include <linux/delay.h> 7 8#include <linux/pds/pds_common.h> 9#include <linux/pds/pds_core_if.h> 10#include <linux/pds/pds_adminq.h> 11 12#include "vfio_dev.h" 13#include "cmds.h" 14 15#define SUSPEND_TIMEOUT_S 5 16#define SUSPEND_CHECK_INTERVAL_MS 1 17 18static int pds_vfio_client_adminq_cmd(struct pds_vfio_pci_device *pds_vfio, 19 union pds_core_adminq_cmd *req, 20 union pds_core_adminq_comp *resp, 21 bool fast_poll) 22{ 23 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio); 24 union pds_core_adminq_cmd cmd = {}; 25 struct pdsc *pdsc; 26 int err; 27 28 /* Wrap the client request */ 29 cmd.client_request.opcode = PDS_AQ_CMD_CLIENT_CMD; 30 cmd.client_request.client_id = cpu_to_le16(pds_vfio->client_id); 31 memcpy(cmd.client_request.client_cmd, req, 32 sizeof(cmd.client_request.client_cmd)); 33 34 pdsc = pdsc_get_pf_struct(pdev); 35 if (IS_ERR(pdsc)) 36 return PTR_ERR(pdsc); 37 38 err = pdsc_adminq_post(pdsc, &cmd, resp, fast_poll); 39 if (err && err != -EAGAIN) 40 dev_err(pds_vfio_to_dev(pds_vfio), 41 "client admin cmd failed: %pe\n", ERR_PTR(err)); 42 43 return err; 44} 45 46int pds_vfio_register_client_cmd(struct pds_vfio_pci_device *pds_vfio) 47{ 48 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio); 49 char devname[PDS_DEVNAME_LEN]; 50 struct pdsc *pdsc; 51 int ci; 52 53 snprintf(devname, sizeof(devname), "%s.%d-%u", PDS_VFIO_LM_DEV_NAME, 54 pci_domain_nr(pdev->bus), 55 PCI_DEVID(pdev->bus->number, pdev->devfn)); 56 57 pdsc = pdsc_get_pf_struct(pdev); 58 if (IS_ERR(pdsc)) 59 return PTR_ERR(pdsc); 60 61 ci = pds_client_register(pdsc, devname); 62 if (ci < 0) 63 return ci; 64 65 pds_vfio->client_id = ci; 66 67 return 0; 68} 69 70void pds_vfio_unregister_client_cmd(struct pds_vfio_pci_device *pds_vfio) 71{ 72 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio); 73 struct pdsc *pdsc; 74 int err; 75 76 pdsc = pdsc_get_pf_struct(pdev); 77 if (IS_ERR(pdsc)) 78 return; 79 80 err = pds_client_unregister(pdsc, pds_vfio->client_id); 81 if (err) 82 dev_err(&pdev->dev, "unregister from DSC failed: %pe\n", 83 ERR_PTR(err)); 84 85 pds_vfio->client_id = 0; 86} 87 88static int 89pds_vfio_suspend_wait_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type) 90{ 91 union pds_core_adminq_cmd cmd = { 92 .lm_suspend_status = { 93 .opcode = PDS_LM_CMD_SUSPEND_STATUS, 94 .vf_id = cpu_to_le16(pds_vfio->vf_id), 95 .type = type, 96 }, 97 }; 98 struct device *dev = pds_vfio_to_dev(pds_vfio); 99 union pds_core_adminq_comp comp = {}; 100 unsigned long time_limit; 101 unsigned long time_start; 102 unsigned long time_done; 103 int err; 104 105 time_start = jiffies; 106 time_limit = time_start + HZ * SUSPEND_TIMEOUT_S; 107 do { 108 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true); 109 if (err != -EAGAIN) 110 break; 111 112 msleep(SUSPEND_CHECK_INTERVAL_MS); 113 } while (time_before(jiffies, time_limit)); 114 115 time_done = jiffies; 116 dev_dbg(dev, "%s: vf%u: Suspend comp received in %d msecs\n", __func__, 117 pds_vfio->vf_id, jiffies_to_msecs(time_done - time_start)); 118 119 /* Check the results */ 120 if (time_after_eq(time_done, time_limit)) { 121 dev_err(dev, "%s: vf%u: Suspend comp timeout\n", __func__, 122 pds_vfio->vf_id); 123 err = -ETIMEDOUT; 124 } 125 126 return err; 127} 128 129int pds_vfio_suspend_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type) 130{ 131 union pds_core_adminq_cmd cmd = { 132 .lm_suspend = { 133 .opcode = PDS_LM_CMD_SUSPEND, 134 .vf_id = cpu_to_le16(pds_vfio->vf_id), 135 .type = type, 136 }, 137 }; 138 struct device *dev = pds_vfio_to_dev(pds_vfio); 139 union pds_core_adminq_comp comp = {}; 140 int err; 141 142 dev_dbg(dev, "vf%u: Suspend device\n", pds_vfio->vf_id); 143 144 /* 145 * The initial suspend request to the firmware starts the device suspend 146 * operation and the firmware returns success if it's started 147 * successfully. 148 */ 149 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true); 150 if (err) { 151 dev_err(dev, "vf%u: Suspend failed: %pe\n", pds_vfio->vf_id, 152 ERR_PTR(err)); 153 return err; 154 } 155 156 /* 157 * The subsequent suspend status request(s) check if the firmware has 158 * completed the device suspend process. 159 */ 160 return pds_vfio_suspend_wait_device_cmd(pds_vfio, type); 161} 162 163int pds_vfio_resume_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type) 164{ 165 union pds_core_adminq_cmd cmd = { 166 .lm_resume = { 167 .opcode = PDS_LM_CMD_RESUME, 168 .vf_id = cpu_to_le16(pds_vfio->vf_id), 169 .type = type, 170 }, 171 }; 172 struct device *dev = pds_vfio_to_dev(pds_vfio); 173 union pds_core_adminq_comp comp = {}; 174 175 dev_dbg(dev, "vf%u: Resume device\n", pds_vfio->vf_id); 176 177 return pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true); 178} 179 180int pds_vfio_get_lm_state_size_cmd(struct pds_vfio_pci_device *pds_vfio, u64 *size) 181{ 182 union pds_core_adminq_cmd cmd = { 183 .lm_state_size = { 184 .opcode = PDS_LM_CMD_STATE_SIZE, 185 .vf_id = cpu_to_le16(pds_vfio->vf_id), 186 }, 187 }; 188 struct device *dev = pds_vfio_to_dev(pds_vfio); 189 union pds_core_adminq_comp comp = {}; 190 int err; 191 192 dev_dbg(dev, "vf%u: Get migration status\n", pds_vfio->vf_id); 193 194 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false); 195 if (err) 196 return err; 197 198 *size = le64_to_cpu(comp.lm_state_size.size); 199 return 0; 200} 201 202static int pds_vfio_dma_map_lm_file(struct device *dev, 203 enum dma_data_direction dir, 204 struct pds_vfio_lm_file *lm_file) 205{ 206 struct pds_lm_sg_elem *sgl, *sge; 207 struct scatterlist *sg; 208 dma_addr_t sgl_addr; 209 size_t sgl_size; 210 int err; 211 int i; 212 213 if (!lm_file) 214 return -EINVAL; 215 216 /* dma map file pages */ 217 err = dma_map_sgtable(dev, &lm_file->sg_table, dir, 0); 218 if (err) 219 return err; 220 221 lm_file->num_sge = lm_file->sg_table.nents; 222 223 /* alloc sgl */ 224 sgl_size = lm_file->num_sge * sizeof(struct pds_lm_sg_elem); 225 sgl = kzalloc(sgl_size, GFP_KERNEL); 226 if (!sgl) { 227 err = -ENOMEM; 228 goto out_unmap_sgtable; 229 } 230 231 /* fill sgl */ 232 sge = sgl; 233 for_each_sgtable_dma_sg(&lm_file->sg_table, sg, i) { 234 sge->addr = cpu_to_le64(sg_dma_address(sg)); 235 sge->len = cpu_to_le32(sg_dma_len(sg)); 236 dev_dbg(dev, "addr = %llx, len = %u\n", sge->addr, sge->len); 237 sge++; 238 } 239 240 sgl_addr = dma_map_single(dev, sgl, sgl_size, DMA_TO_DEVICE); 241 if (dma_mapping_error(dev, sgl_addr)) { 242 err = -EIO; 243 goto out_free_sgl; 244 } 245 246 lm_file->sgl = sgl; 247 lm_file->sgl_addr = sgl_addr; 248 249 return 0; 250 251out_free_sgl: 252 kfree(sgl); 253out_unmap_sgtable: 254 lm_file->num_sge = 0; 255 dma_unmap_sgtable(dev, &lm_file->sg_table, dir, 0); 256 return err; 257} 258 259static void pds_vfio_dma_unmap_lm_file(struct device *dev, 260 enum dma_data_direction dir, 261 struct pds_vfio_lm_file *lm_file) 262{ 263 if (!lm_file) 264 return; 265 266 /* free sgl */ 267 if (lm_file->sgl) { 268 dma_unmap_single(dev, lm_file->sgl_addr, 269 lm_file->num_sge * sizeof(*lm_file->sgl), 270 DMA_TO_DEVICE); 271 kfree(lm_file->sgl); 272 lm_file->sgl = NULL; 273 lm_file->sgl_addr = DMA_MAPPING_ERROR; 274 lm_file->num_sge = 0; 275 } 276 277 /* dma unmap file pages */ 278 dma_unmap_sgtable(dev, &lm_file->sg_table, dir, 0); 279} 280 281int pds_vfio_get_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio) 282{ 283 union pds_core_adminq_cmd cmd = { 284 .lm_save = { 285 .opcode = PDS_LM_CMD_SAVE, 286 .vf_id = cpu_to_le16(pds_vfio->vf_id), 287 }, 288 }; 289 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio); 290 struct device *pdsc_dev = &pci_physfn(pdev)->dev; 291 union pds_core_adminq_comp comp = {}; 292 struct pds_vfio_lm_file *lm_file; 293 int err; 294 295 dev_dbg(&pdev->dev, "vf%u: Get migration state\n", pds_vfio->vf_id); 296 297 lm_file = pds_vfio->save_file; 298 299 err = pds_vfio_dma_map_lm_file(pdsc_dev, DMA_FROM_DEVICE, lm_file); 300 if (err) { 301 dev_err(&pdev->dev, "failed to map save migration file: %pe\n", 302 ERR_PTR(err)); 303 return err; 304 } 305 306 cmd.lm_save.sgl_addr = cpu_to_le64(lm_file->sgl_addr); 307 cmd.lm_save.num_sge = cpu_to_le32(lm_file->num_sge); 308 309 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false); 310 if (err) 311 dev_err(&pdev->dev, "failed to get migration state: %pe\n", 312 ERR_PTR(err)); 313 314 pds_vfio_dma_unmap_lm_file(pdsc_dev, DMA_FROM_DEVICE, lm_file); 315 316 return err; 317} 318 319int pds_vfio_set_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio) 320{ 321 union pds_core_adminq_cmd cmd = { 322 .lm_restore = { 323 .opcode = PDS_LM_CMD_RESTORE, 324 .vf_id = cpu_to_le16(pds_vfio->vf_id), 325 }, 326 }; 327 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio); 328 struct device *pdsc_dev = &pci_physfn(pdev)->dev; 329 union pds_core_adminq_comp comp = {}; 330 struct pds_vfio_lm_file *lm_file; 331 int err; 332 333 dev_dbg(&pdev->dev, "vf%u: Set migration state\n", pds_vfio->vf_id); 334 335 lm_file = pds_vfio->restore_file; 336 337 err = pds_vfio_dma_map_lm_file(pdsc_dev, DMA_TO_DEVICE, lm_file); 338 if (err) { 339 dev_err(&pdev->dev, 340 "failed to map restore migration file: %pe\n", 341 ERR_PTR(err)); 342 return err; 343 } 344 345 cmd.lm_restore.sgl_addr = cpu_to_le64(lm_file->sgl_addr); 346 cmd.lm_restore.num_sge = cpu_to_le32(lm_file->num_sge); 347 348 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false); 349 if (err) 350 dev_err(&pdev->dev, "failed to set migration state: %pe\n", 351 ERR_PTR(err)); 352 353 pds_vfio_dma_unmap_lm_file(pdsc_dev, DMA_TO_DEVICE, lm_file); 354 355 return err; 356} 357 358void pds_vfio_send_host_vf_lm_status_cmd(struct pds_vfio_pci_device *pds_vfio, 359 enum pds_lm_host_vf_status vf_status) 360{ 361 union pds_core_adminq_cmd cmd = { 362 .lm_host_vf_status = { 363 .opcode = PDS_LM_CMD_HOST_VF_STATUS, 364 .vf_id = cpu_to_le16(pds_vfio->vf_id), 365 .status = vf_status, 366 }, 367 }; 368 struct device *dev = pds_vfio_to_dev(pds_vfio); 369 union pds_core_adminq_comp comp = {}; 370 int err; 371 372 dev_dbg(dev, "vf%u: Set host VF LM status: %u", pds_vfio->vf_id, 373 vf_status); 374 if (vf_status != PDS_LM_STA_IN_PROGRESS && 375 vf_status != PDS_LM_STA_NONE) { 376 dev_warn(dev, "Invalid host VF migration status, %d\n", 377 vf_status); 378 return; 379 } 380 381 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false); 382 if (err) 383 dev_warn(dev, "failed to send host VF migration status: %pe\n", 384 ERR_PTR(err)); 385} 386 387int pds_vfio_dirty_status_cmd(struct pds_vfio_pci_device *pds_vfio, 388 u64 regions_dma, u8 *max_regions, u8 *num_regions) 389{ 390 union pds_core_adminq_cmd cmd = { 391 .lm_dirty_status = { 392 .opcode = PDS_LM_CMD_DIRTY_STATUS, 393 .vf_id = cpu_to_le16(pds_vfio->vf_id), 394 }, 395 }; 396 struct device *dev = pds_vfio_to_dev(pds_vfio); 397 union pds_core_adminq_comp comp = {}; 398 int err; 399 400 dev_dbg(dev, "vf%u: Dirty status\n", pds_vfio->vf_id); 401 402 cmd.lm_dirty_status.regions_dma = cpu_to_le64(regions_dma); 403 cmd.lm_dirty_status.max_regions = *max_regions; 404 405 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false); 406 if (err) { 407 dev_err(dev, "failed to get dirty status: %pe\n", ERR_PTR(err)); 408 return err; 409 } 410 411 /* only support seq_ack approach for now */ 412 if (!(le32_to_cpu(comp.lm_dirty_status.bmp_type_mask) & 413 BIT(PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK))) { 414 dev_err(dev, "Dirty bitmap tracking SEQ_ACK not supported\n"); 415 return -EOPNOTSUPP; 416 } 417 418 *num_regions = comp.lm_dirty_status.num_regions; 419 *max_regions = comp.lm_dirty_status.max_regions; 420 421 dev_dbg(dev, 422 "Page Tracking Status command successful, max_regions: %d, num_regions: %d, bmp_type: %s\n", 423 *max_regions, *num_regions, "PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK"); 424 425 return 0; 426} 427 428int pds_vfio_dirty_enable_cmd(struct pds_vfio_pci_device *pds_vfio, 429 u64 regions_dma, u8 num_regions) 430{ 431 union pds_core_adminq_cmd cmd = { 432 .lm_dirty_enable = { 433 .opcode = PDS_LM_CMD_DIRTY_ENABLE, 434 .vf_id = cpu_to_le16(pds_vfio->vf_id), 435 .regions_dma = cpu_to_le64(regions_dma), 436 .bmp_type = PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK, 437 .num_regions = num_regions, 438 }, 439 }; 440 struct device *dev = pds_vfio_to_dev(pds_vfio); 441 union pds_core_adminq_comp comp = {}; 442 int err; 443 444 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false); 445 if (err) { 446 dev_err(dev, "failed dirty tracking enable: %pe\n", 447 ERR_PTR(err)); 448 return err; 449 } 450 451 return 0; 452} 453 454int pds_vfio_dirty_disable_cmd(struct pds_vfio_pci_device *pds_vfio) 455{ 456 union pds_core_adminq_cmd cmd = { 457 .lm_dirty_disable = { 458 .opcode = PDS_LM_CMD_DIRTY_DISABLE, 459 .vf_id = cpu_to_le16(pds_vfio->vf_id), 460 }, 461 }; 462 struct device *dev = pds_vfio_to_dev(pds_vfio); 463 union pds_core_adminq_comp comp = {}; 464 int err; 465 466 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false); 467 if (err || comp.lm_dirty_status.num_regions != 0) { 468 /* in case num_regions is still non-zero after disable */ 469 err = err ? err : -EIO; 470 dev_err(dev, 471 "failed dirty tracking disable: %pe, num_regions %d\n", 472 ERR_PTR(err), comp.lm_dirty_status.num_regions); 473 return err; 474 } 475 476 return 0; 477} 478 479int pds_vfio_dirty_seq_ack_cmd(struct pds_vfio_pci_device *pds_vfio, 480 u64 sgl_dma, u16 num_sge, u32 offset, 481 u32 total_len, bool read_seq) 482{ 483 const char *cmd_type_str = read_seq ? "read_seq" : "write_ack"; 484 union pds_core_adminq_cmd cmd = { 485 .lm_dirty_seq_ack = { 486 .vf_id = cpu_to_le16(pds_vfio->vf_id), 487 .len_bytes = cpu_to_le32(total_len), 488 .off_bytes = cpu_to_le32(offset), 489 .sgl_addr = cpu_to_le64(sgl_dma), 490 .num_sge = cpu_to_le16(num_sge), 491 }, 492 }; 493 struct device *dev = pds_vfio_to_dev(pds_vfio); 494 union pds_core_adminq_comp comp = {}; 495 int err; 496 497 if (read_seq) 498 cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_READ_SEQ; 499 else 500 cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_WRITE_ACK; 501 502 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false); 503 if (err) { 504 dev_err(dev, "failed cmd Page Tracking %s: %pe\n", cmd_type_str, 505 ERR_PTR(err)); 506 return err; 507 } 508 509 return 0; 510} 511