182498Sroberto// SPDX-License-Identifier: GPL-2.0 282498Sroberto/* Copyright(c) 2023 Advanced Micro Devices, Inc. */ 382498Sroberto 4280849Scy#include <linux/interval_tree.h> 5280849Scy#include <linux/vfio.h> 6280849Scy 7280849Scy#include <linux/pds/pds_common.h> 8280849Scy#include <linux/pds/pds_core_if.h> 9280849Scy#include <linux/pds/pds_adminq.h> 10280849Scy 11280849Scy#include "vfio_dev.h" 12280849Scy#include "cmds.h" 13280849Scy#include "dirty.h" 14280849Scy 15280849Scy#define READ_SEQ true 16280849Scy#define WRITE_ACK false 17280849Scy 18280849Scybool pds_vfio_dirty_is_enabled(struct pds_vfio_pci_device *pds_vfio) 19280849Scy{ 20280849Scy return pds_vfio->dirty.is_enabled; 21280849Scy} 22280849Scy 23280849Scyvoid pds_vfio_dirty_set_enabled(struct pds_vfio_pci_device *pds_vfio) 24280849Scy{ 25280849Scy pds_vfio->dirty.is_enabled = true; 26280849Scy} 27316068Sdelphij 28132451Srobertovoid pds_vfio_dirty_set_disabled(struct pds_vfio_pci_device *pds_vfio) 29280849Scy{ 30280849Scy pds_vfio->dirty.is_enabled = false; 31280849Scy} 32132451Sroberto 33132451Srobertostatic void 34132451Srobertopds_vfio_print_guest_region_info(struct pds_vfio_pci_device *pds_vfio, 35132451Sroberto u8 max_regions) 36132451Sroberto{ 37132451Sroberto int len = max_regions * sizeof(struct pds_lm_dirty_region_info); 3882498Sroberto struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev; 39132451Sroberto struct device *pdsc_dev = &pci_physfn(pdev)->dev; 40132451Sroberto struct pds_lm_dirty_region_info *region_info; 41132451Sroberto dma_addr_t regions_dma; 42132451Sroberto u8 num_regions; 43132451Sroberto int err; 44132451Sroberto 45132451Sroberto region_info = kcalloc(max_regions, 46132451Sroberto sizeof(struct pds_lm_dirty_region_info), 47132451Sroberto GFP_KERNEL); 48132451Sroberto if (!region_info) 49280849Scy return; 50132451Sroberto 51132451Sroberto regions_dma = 52280849Scy dma_map_single(pdsc_dev, region_info, len, DMA_FROM_DEVICE); 53132451Sroberto if (dma_mapping_error(pdsc_dev, regions_dma)) 54132451Sroberto goto out_free_region_info; 55280849Scy 56280849Scy err = pds_vfio_dirty_status_cmd(pds_vfio, regions_dma, &max_regions, 5782498Sroberto &num_regions); 5882498Sroberto dma_unmap_single(pdsc_dev, regions_dma, len, DMA_FROM_DEVICE); 59132451Sroberto if (err) 6082498Sroberto goto out_free_region_info; 61182007Sroberto 62182007Sroberto for (unsigned int i = 0; i < num_regions; i++) 63182007Sroberto dev_dbg(&pdev->dev, 64182007Sroberto "region_info[%d]: dma_base 0x%llx page_count %u page_size_log2 %u\n", 65132451Sroberto i, le64_to_cpu(region_info[i].dma_base), 6682498Sroberto le32_to_cpu(region_info[i].page_count), 6782498Sroberto region_info[i].page_size_log2); 6882498Sroberto 6982498Srobertoout_free_region_info: 70132451Sroberto kfree(region_info); 71132451Sroberto} 72132451Sroberto 73132451Srobertostatic int pds_vfio_dirty_alloc_bitmaps(struct pds_vfio_region *region, 74132451Sroberto unsigned long bytes) 75132451Sroberto{ 76132451Sroberto unsigned long *host_seq_bmp, *host_ack_bmp; 77132451Sroberto 78280849Scy host_seq_bmp = vzalloc(bytes); 79132451Sroberto if (!host_seq_bmp) 80132451Sroberto return -ENOMEM; 81132451Sroberto 82132451Sroberto host_ack_bmp = vzalloc(bytes); 83132451Sroberto if (!host_ack_bmp) { 84132451Sroberto bitmap_free(host_seq_bmp); 8582498Sroberto return -ENOMEM; 86132451Sroberto } 87132451Sroberto 88132451Sroberto region->host_seq = host_seq_bmp; 89132451Sroberto region->host_ack = host_ack_bmp; 90132451Sroberto region->bmp_bytes = bytes; 91132451Sroberto 92132451Sroberto return 0; 93132451Sroberto} 94182007Sroberto 95132451Srobertostatic void pds_vfio_dirty_free_bitmaps(struct pds_vfio_dirty *dirty) 96132451Sroberto{ 97132451Sroberto if (!dirty->regions) 98132451Sroberto return; 99132451Sroberto 100182007Sroberto for (int i = 0; i < dirty->num_regions; i++) { 101132451Sroberto struct pds_vfio_region *region = &dirty->regions[i]; 102280849Scy 103132451Sroberto vfree(region->host_seq); 104182007Sroberto vfree(region->host_ack); 105182007Sroberto region->host_seq = NULL; 10682498Sroberto region->host_ack = NULL; 10782498Sroberto region->bmp_bytes = 0; 108132451Sroberto } 10982498Sroberto} 110132451Sroberto 111280849Scystatic void __pds_vfio_dirty_free_sgl(struct pds_vfio_pci_device *pds_vfio, 112280849Scy struct pds_vfio_region *region) 113280849Scy{ 114280849Scy struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev; 115280849Scy struct device *pdsc_dev = &pci_physfn(pdev)->dev; 11682498Sroberto 11782498Sroberto dma_unmap_single(pdsc_dev, region->sgl_addr, 118132451Sroberto region->num_sge * sizeof(struct pds_lm_sg_elem), 119132451Sroberto DMA_BIDIRECTIONAL); 120132451Sroberto kfree(region->sgl); 121132451Sroberto 122132451Sroberto region->num_sge = 0; 123132451Sroberto region->sgl = NULL; 124132451Sroberto region->sgl_addr = 0; 125132451Sroberto} 126132451Sroberto 127132451Srobertostatic void pds_vfio_dirty_free_sgl(struct pds_vfio_pci_device *pds_vfio) 128132451Sroberto{ 129132451Sroberto struct pds_vfio_dirty *dirty = &pds_vfio->dirty; 130132451Sroberto 131132451Sroberto if (!dirty->regions) 132132451Sroberto return; 133132451Sroberto 134280849Scy for (int i = 0; i < dirty->num_regions; i++) { 135132451Sroberto struct pds_vfio_region *region = &dirty->regions[i]; 136132451Sroberto 137132451Sroberto if (region->sgl) 138132451Sroberto __pds_vfio_dirty_free_sgl(pds_vfio, region); 139132451Sroberto } 140132451Sroberto} 141132451Sroberto 142132451Srobertostatic int pds_vfio_dirty_alloc_sgl(struct pds_vfio_pci_device *pds_vfio, 143132451Sroberto struct pds_vfio_region *region, 144132451Sroberto u32 page_count) 145132451Sroberto{ 146132451Sroberto struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev; 147132451Sroberto struct device *pdsc_dev = &pci_physfn(pdev)->dev; 148132451Sroberto struct pds_lm_sg_elem *sgl; 149132451Sroberto dma_addr_t sgl_addr; 150132451Sroberto size_t sgl_size; 151132451Sroberto u32 max_sge; 152280849Scy 153132451Sroberto max_sge = DIV_ROUND_UP(page_count, PAGE_SIZE * 8); 154132451Sroberto sgl_size = max_sge * sizeof(struct pds_lm_sg_elem); 155132451Sroberto 156132451Sroberto sgl = kzalloc(sgl_size, GFP_KERNEL); 157132451Sroberto if (!sgl) 158132451Sroberto return -ENOMEM; 159132451Sroberto 160132451Sroberto sgl_addr = dma_map_single(pdsc_dev, sgl, sgl_size, DMA_BIDIRECTIONAL); 161132451Sroberto if (dma_mapping_error(pdsc_dev, sgl_addr)) { 162132451Sroberto kfree(sgl); 163132451Sroberto return -EIO; 164280849Scy } 165280849Scy 166132451Sroberto region->sgl = sgl; 167132451Sroberto region->num_sge = max_sge; 168280849Scy region->sgl_addr = sgl_addr; 169132451Sroberto 170132451Sroberto return 0; 171132451Sroberto} 172132451Sroberto 173280849Scystatic void pds_vfio_dirty_free_regions(struct pds_vfio_dirty *dirty) 174280849Scy{ 175280849Scy vfree(dirty->regions); 176280849Scy dirty->regions = NULL; 177280849Scy dirty->num_regions = 0; 178280849Scy} 179280849Scy 180280849Scystatic int pds_vfio_dirty_alloc_regions(struct pds_vfio_pci_device *pds_vfio, 181280849Scy struct pds_lm_dirty_region_info *region_info, 182280849Scy u64 region_page_size, u8 num_regions) 18382498Sroberto{ 18482498Sroberto struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev; 18582498Sroberto struct pds_vfio_dirty *dirty = &pds_vfio->dirty; 186280849Scy u32 dev_bmp_offset_byte = 0; 187132451Sroberto int err; 188132451Sroberto 18982498Sroberto dirty->regions = vcalloc(num_regions, sizeof(struct pds_vfio_region)); 190280849Scy if (!dirty->regions) 191280849Scy return -ENOMEM; 192 dirty->num_regions = num_regions; 193 194 for (int i = 0; i < num_regions; i++) { 195 struct pds_lm_dirty_region_info *ri = ®ion_info[i]; 196 struct pds_vfio_region *region = &dirty->regions[i]; 197 u64 region_size, region_start; 198 u32 page_count; 199 200 /* page_count might be adjusted by the device */ 201 page_count = le32_to_cpu(ri->page_count); 202 region_start = le64_to_cpu(ri->dma_base); 203 region_size = page_count * region_page_size; 204 205 err = pds_vfio_dirty_alloc_bitmaps(region, 206 page_count / BITS_PER_BYTE); 207 if (err) { 208 dev_err(&pdev->dev, "Failed to alloc dirty bitmaps: %pe\n", 209 ERR_PTR(err)); 210 goto out_free_regions; 211 } 212 213 err = pds_vfio_dirty_alloc_sgl(pds_vfio, region, page_count); 214 if (err) { 215 dev_err(&pdev->dev, "Failed to alloc dirty sg lists: %pe\n", 216 ERR_PTR(err)); 217 goto out_free_regions; 218 } 219 220 region->size = region_size; 221 region->start = region_start; 222 region->page_size = region_page_size; 223 region->dev_bmp_offset_start_byte = dev_bmp_offset_byte; 224 225 dev_bmp_offset_byte += page_count / BITS_PER_BYTE; 226 if (dev_bmp_offset_byte % BITS_PER_BYTE) { 227 dev_err(&pdev->dev, "Device bitmap offset is mis-aligned\n"); 228 err = -EINVAL; 229 goto out_free_regions; 230 } 231 } 232 233 return 0; 234 235out_free_regions: 236 pds_vfio_dirty_free_bitmaps(dirty); 237 pds_vfio_dirty_free_sgl(pds_vfio); 238 pds_vfio_dirty_free_regions(dirty); 239 240 return err; 241} 242 243static int pds_vfio_dirty_enable(struct pds_vfio_pci_device *pds_vfio, 244 struct rb_root_cached *ranges, u32 nnodes, 245 u64 *page_size) 246{ 247 struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev; 248 struct device *pdsc_dev = &pci_physfn(pdev)->dev; 249 struct pds_lm_dirty_region_info *region_info; 250 struct interval_tree_node *node = NULL; 251 u64 region_page_size = *page_size; 252 u8 max_regions = 0, num_regions; 253 dma_addr_t regions_dma = 0; 254 u32 num_ranges = nnodes; 255 int err; 256 u16 len; 257 258 dev_dbg(&pdev->dev, "vf%u: Start dirty page tracking\n", 259 pds_vfio->vf_id); 260 261 if (pds_vfio_dirty_is_enabled(pds_vfio)) 262 return -EINVAL; 263 264 /* find if dirty tracking is disabled, i.e. num_regions == 0 */ 265 err = pds_vfio_dirty_status_cmd(pds_vfio, 0, &max_regions, 266 &num_regions); 267 if (err < 0) { 268 dev_err(&pdev->dev, "Failed to get dirty status, err %pe\n", 269 ERR_PTR(err)); 270 return err; 271 } else if (num_regions) { 272 dev_err(&pdev->dev, 273 "Dirty tracking already enabled for %d regions\n", 274 num_regions); 275 return -EEXIST; 276 } else if (!max_regions) { 277 dev_err(&pdev->dev, 278 "Device doesn't support dirty tracking, max_regions %d\n", 279 max_regions); 280 return -EOPNOTSUPP; 281 } 282 283 if (num_ranges > max_regions) { 284 vfio_combine_iova_ranges(ranges, nnodes, max_regions); 285 num_ranges = max_regions; 286 } 287 288 region_info = kcalloc(num_ranges, sizeof(*region_info), GFP_KERNEL); 289 if (!region_info) 290 return -ENOMEM; 291 len = num_ranges * sizeof(*region_info); 292 293 node = interval_tree_iter_first(ranges, 0, ULONG_MAX); 294 if (!node) 295 return -EINVAL; 296 for (int i = 0; i < num_ranges; i++) { 297 struct pds_lm_dirty_region_info *ri = ®ion_info[i]; 298 u64 region_size = node->last - node->start + 1; 299 u64 region_start = node->start; 300 u32 page_count; 301 302 page_count = DIV_ROUND_UP(region_size, region_page_size); 303 304 ri->dma_base = cpu_to_le64(region_start); 305 ri->page_count = cpu_to_le32(page_count); 306 ri->page_size_log2 = ilog2(region_page_size); 307 308 dev_dbg(&pdev->dev, 309 "region_info[%d]: region_start 0x%llx region_end 0x%lx region_size 0x%llx page_count %u page_size %llu\n", 310 i, region_start, node->last, region_size, page_count, 311 region_page_size); 312 313 node = interval_tree_iter_next(node, 0, ULONG_MAX); 314 } 315 316 regions_dma = dma_map_single(pdsc_dev, (void *)region_info, len, 317 DMA_BIDIRECTIONAL); 318 if (dma_mapping_error(pdsc_dev, regions_dma)) { 319 err = -ENOMEM; 320 goto out_free_region_info; 321 } 322 323 err = pds_vfio_dirty_enable_cmd(pds_vfio, regions_dma, num_ranges); 324 dma_unmap_single(pdsc_dev, regions_dma, len, DMA_BIDIRECTIONAL); 325 if (err) 326 goto out_free_region_info; 327 328 err = pds_vfio_dirty_alloc_regions(pds_vfio, region_info, 329 region_page_size, num_ranges); 330 if (err) { 331 dev_err(&pdev->dev, 332 "Failed to allocate %d regions for tracking dirty regions: %pe\n", 333 num_regions, ERR_PTR(err)); 334 goto out_dirty_disable; 335 } 336 337 pds_vfio_dirty_set_enabled(pds_vfio); 338 339 pds_vfio_print_guest_region_info(pds_vfio, max_regions); 340 341 kfree(region_info); 342 343 return 0; 344 345out_dirty_disable: 346 pds_vfio_dirty_disable_cmd(pds_vfio); 347out_free_region_info: 348 kfree(region_info); 349 return err; 350} 351 352void pds_vfio_dirty_disable(struct pds_vfio_pci_device *pds_vfio, bool send_cmd) 353{ 354 if (pds_vfio_dirty_is_enabled(pds_vfio)) { 355 pds_vfio_dirty_set_disabled(pds_vfio); 356 if (send_cmd) 357 pds_vfio_dirty_disable_cmd(pds_vfio); 358 pds_vfio_dirty_free_sgl(pds_vfio); 359 pds_vfio_dirty_free_bitmaps(&pds_vfio->dirty); 360 pds_vfio_dirty_free_regions(&pds_vfio->dirty); 361 } 362 363 if (send_cmd) 364 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE); 365} 366 367static int pds_vfio_dirty_seq_ack(struct pds_vfio_pci_device *pds_vfio, 368 struct pds_vfio_region *region, 369 unsigned long *seq_ack_bmp, u32 offset, 370 u32 bmp_bytes, bool read_seq) 371{ 372 const char *bmp_type_str = read_seq ? "read_seq" : "write_ack"; 373 u8 dma_dir = read_seq ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 374 struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev; 375 struct device *pdsc_dev = &pci_physfn(pdev)->dev; 376 unsigned long long npages; 377 struct sg_table sg_table; 378 struct scatterlist *sg; 379 struct page **pages; 380 u32 page_offset; 381 const void *bmp; 382 size_t size; 383 u16 num_sge; 384 int err; 385 int i; 386 387 bmp = (void *)((u64)seq_ack_bmp + offset); 388 page_offset = offset_in_page(bmp); 389 bmp -= page_offset; 390 391 /* 392 * Start and end of bitmap section to seq/ack might not be page 393 * aligned, so use the page_offset to account for that so there 394 * will be enough pages to represent the bmp_bytes 395 */ 396 npages = DIV_ROUND_UP_ULL(bmp_bytes + page_offset, PAGE_SIZE); 397 pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL); 398 if (!pages) 399 return -ENOMEM; 400 401 for (unsigned long long i = 0; i < npages; i++) { 402 struct page *page = vmalloc_to_page(bmp); 403 404 if (!page) { 405 err = -EFAULT; 406 goto out_free_pages; 407 } 408 409 pages[i] = page; 410 bmp += PAGE_SIZE; 411 } 412 413 err = sg_alloc_table_from_pages(&sg_table, pages, npages, page_offset, 414 bmp_bytes, GFP_KERNEL); 415 if (err) 416 goto out_free_pages; 417 418 err = dma_map_sgtable(pdsc_dev, &sg_table, dma_dir, 0); 419 if (err) 420 goto out_free_sg_table; 421 422 for_each_sgtable_dma_sg(&sg_table, sg, i) { 423 struct pds_lm_sg_elem *sg_elem = ®ion->sgl[i]; 424 425 sg_elem->addr = cpu_to_le64(sg_dma_address(sg)); 426 sg_elem->len = cpu_to_le32(sg_dma_len(sg)); 427 } 428 429 num_sge = sg_table.nents; 430 size = num_sge * sizeof(struct pds_lm_sg_elem); 431 offset += region->dev_bmp_offset_start_byte; 432 dma_sync_single_for_device(pdsc_dev, region->sgl_addr, size, dma_dir); 433 err = pds_vfio_dirty_seq_ack_cmd(pds_vfio, region->sgl_addr, num_sge, 434 offset, bmp_bytes, read_seq); 435 if (err) 436 dev_err(&pdev->dev, 437 "Dirty bitmap %s failed offset %u bmp_bytes %u num_sge %u DMA 0x%llx: %pe\n", 438 bmp_type_str, offset, bmp_bytes, 439 num_sge, region->sgl_addr, ERR_PTR(err)); 440 dma_sync_single_for_cpu(pdsc_dev, region->sgl_addr, size, dma_dir); 441 442 dma_unmap_sgtable(pdsc_dev, &sg_table, dma_dir, 0); 443out_free_sg_table: 444 sg_free_table(&sg_table); 445out_free_pages: 446 kfree(pages); 447 448 return err; 449} 450 451static int pds_vfio_dirty_write_ack(struct pds_vfio_pci_device *pds_vfio, 452 struct pds_vfio_region *region, 453 u32 offset, u32 len) 454{ 455 456 return pds_vfio_dirty_seq_ack(pds_vfio, region, region->host_ack, 457 offset, len, WRITE_ACK); 458} 459 460static int pds_vfio_dirty_read_seq(struct pds_vfio_pci_device *pds_vfio, 461 struct pds_vfio_region *region, 462 u32 offset, u32 len) 463{ 464 return pds_vfio_dirty_seq_ack(pds_vfio, region, region->host_seq, 465 offset, len, READ_SEQ); 466} 467 468static int pds_vfio_dirty_process_bitmaps(struct pds_vfio_pci_device *pds_vfio, 469 struct pds_vfio_region *region, 470 struct iova_bitmap *dirty_bitmap, 471 u32 bmp_offset, u32 len_bytes) 472{ 473 u64 page_size = region->page_size; 474 u64 region_start = region->start; 475 u32 bmp_offset_bit; 476 __le64 *seq, *ack; 477 int dword_count; 478 479 dword_count = len_bytes / sizeof(u64); 480 seq = (__le64 *)((u64)region->host_seq + bmp_offset); 481 ack = (__le64 *)((u64)region->host_ack + bmp_offset); 482 bmp_offset_bit = bmp_offset * 8; 483 484 for (int i = 0; i < dword_count; i++) { 485 u64 xor = le64_to_cpu(seq[i]) ^ le64_to_cpu(ack[i]); 486 487 /* prepare for next write_ack call */ 488 ack[i] = seq[i]; 489 490 for (u8 bit_i = 0; bit_i < BITS_PER_TYPE(u64); ++bit_i) { 491 if (xor & BIT(bit_i)) { 492 u64 abs_bit_i = bmp_offset_bit + 493 i * BITS_PER_TYPE(u64) + bit_i; 494 u64 addr = abs_bit_i * page_size + region_start; 495 496 iova_bitmap_set(dirty_bitmap, addr, page_size); 497 } 498 } 499 } 500 501 return 0; 502} 503 504static struct pds_vfio_region * 505pds_vfio_get_region(struct pds_vfio_pci_device *pds_vfio, unsigned long iova) 506{ 507 struct pds_vfio_dirty *dirty = &pds_vfio->dirty; 508 509 for (int i = 0; i < dirty->num_regions; i++) { 510 struct pds_vfio_region *region = &dirty->regions[i]; 511 512 if (iova >= region->start && 513 iova < (region->start + region->size)) 514 return region; 515 } 516 517 return NULL; 518} 519 520static int pds_vfio_dirty_sync(struct pds_vfio_pci_device *pds_vfio, 521 struct iova_bitmap *dirty_bitmap, 522 unsigned long iova, unsigned long length) 523{ 524 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev; 525 struct pds_vfio_region *region; 526 u64 bmp_offset, bmp_bytes; 527 u64 bitmap_size, pages; 528 int err; 529 530 dev_dbg(dev, "vf%u: Get dirty page bitmap\n", pds_vfio->vf_id); 531 532 if (!pds_vfio_dirty_is_enabled(pds_vfio)) { 533 dev_err(dev, "vf%u: Sync failed, dirty tracking is disabled\n", 534 pds_vfio->vf_id); 535 return -EINVAL; 536 } 537 538 region = pds_vfio_get_region(pds_vfio, iova); 539 if (!region) { 540 dev_err(dev, "vf%u: Failed to find region that contains iova 0x%lx length 0x%lx\n", 541 pds_vfio->vf_id, iova, length); 542 return -EINVAL; 543 } 544 545 pages = DIV_ROUND_UP(length, region->page_size); 546 bitmap_size = 547 round_up(pages, sizeof(u64) * BITS_PER_BYTE) / BITS_PER_BYTE; 548 549 dev_dbg(dev, 550 "vf%u: iova 0x%lx length %lu page_size %llu pages %llu bitmap_size %llu\n", 551 pds_vfio->vf_id, iova, length, region->page_size, 552 pages, bitmap_size); 553 554 if (!length || ((iova - region->start + length) > region->size)) { 555 dev_err(dev, "Invalid iova 0x%lx and/or length 0x%lx to sync\n", 556 iova, length); 557 return -EINVAL; 558 } 559 560 /* bitmap is modified in 64 bit chunks */ 561 bmp_bytes = ALIGN(DIV_ROUND_UP(length / region->page_size, 562 sizeof(u64)), sizeof(u64)); 563 if (bmp_bytes != bitmap_size) { 564 dev_err(dev, 565 "Calculated bitmap bytes %llu not equal to bitmap size %llu\n", 566 bmp_bytes, bitmap_size); 567 return -EINVAL; 568 } 569 570 if (bmp_bytes > region->bmp_bytes) { 571 dev_err(dev, 572 "Calculated bitmap bytes %llu larger than region's cached bmp_bytes %llu\n", 573 bmp_bytes, region->bmp_bytes); 574 return -EINVAL; 575 } 576 577 bmp_offset = DIV_ROUND_UP((iova - region->start) / 578 region->page_size, sizeof(u64)); 579 580 dev_dbg(dev, 581 "Syncing dirty bitmap, iova 0x%lx length 0x%lx, bmp_offset %llu bmp_bytes %llu\n", 582 iova, length, bmp_offset, bmp_bytes); 583 584 err = pds_vfio_dirty_read_seq(pds_vfio, region, bmp_offset, bmp_bytes); 585 if (err) 586 return err; 587 588 err = pds_vfio_dirty_process_bitmaps(pds_vfio, region, dirty_bitmap, 589 bmp_offset, bmp_bytes); 590 if (err) 591 return err; 592 593 err = pds_vfio_dirty_write_ack(pds_vfio, region, bmp_offset, bmp_bytes); 594 if (err) 595 return err; 596 597 return 0; 598} 599 600int pds_vfio_dma_logging_report(struct vfio_device *vdev, unsigned long iova, 601 unsigned long length, struct iova_bitmap *dirty) 602{ 603 struct pds_vfio_pci_device *pds_vfio = 604 container_of(vdev, struct pds_vfio_pci_device, 605 vfio_coredev.vdev); 606 int err; 607 608 mutex_lock(&pds_vfio->state_mutex); 609 err = pds_vfio_dirty_sync(pds_vfio, dirty, iova, length); 610 mutex_unlock(&pds_vfio->state_mutex); 611 612 return err; 613} 614 615int pds_vfio_dma_logging_start(struct vfio_device *vdev, 616 struct rb_root_cached *ranges, u32 nnodes, 617 u64 *page_size) 618{ 619 struct pds_vfio_pci_device *pds_vfio = 620 container_of(vdev, struct pds_vfio_pci_device, 621 vfio_coredev.vdev); 622 int err; 623 624 mutex_lock(&pds_vfio->state_mutex); 625 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_IN_PROGRESS); 626 err = pds_vfio_dirty_enable(pds_vfio, ranges, nnodes, page_size); 627 mutex_unlock(&pds_vfio->state_mutex); 628 629 return err; 630} 631 632int pds_vfio_dma_logging_stop(struct vfio_device *vdev) 633{ 634 struct pds_vfio_pci_device *pds_vfio = 635 container_of(vdev, struct pds_vfio_pci_device, 636 vfio_coredev.vdev); 637 638 mutex_lock(&pds_vfio->state_mutex); 639 pds_vfio_dirty_disable(pds_vfio, true); 640 mutex_unlock(&pds_vfio->state_mutex); 641 642 return 0; 643} 644