Lines Matching refs:its
28 static int vgic_its_save_tables_v0(struct vgic_its *its);
29 static int vgic_its_restore_tables_v0(struct vgic_its *its);
30 static int vgic_its_commit_v0(struct vgic_its *its);
36 * If this LPI is already mapped on another ITS, we increase its refcount
167 int (*save_tables)(struct vgic_its *its);
168 int (*restore_tables)(struct vgic_its *its);
169 int (*commit)(struct vgic_its *its);
188 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
190 return &its_table_abi_versions[its->abi_rev];
193 static int vgic_its_set_abi(struct vgic_its *its, u32 rev)
197 its->abi_rev = rev;
198 abi = vgic_its_get_abi(its);
199 return abi->commit(its);
206 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
210 list_for_each_entry(device, &its->device_list, dev_list)
222 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
228 device = find_its_device(its, device_id);
240 #define for_each_lpi_its(dev, ite, its) \
241 list_for_each_entry(dev, &(its)->device_list, dev_list) \
257 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
261 list_for_each_entry(collection, &its->collection_list, coll_list) {
366 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
372 for_each_lpi_its(device, ite, its) {
437 struct vgic_its *its,
440 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
459 struct vgic_its *its,
464 val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
470 struct vgic_its *its,
478 return vgic_its_set_abi(its, rev);
482 struct vgic_its *its,
524 return iodev->its;
537 struct vgic_its *its;
543 its = __vgic_doorbell_to_its(kvm, db);
544 if (IS_ERR(its))
549 irq = xa_load(&its->translation_cache, cache_key);
558 static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,
573 lockdep_assert_held(&its->its_lock);
581 old = xa_store(&its->translation_cache, cache_key, irq, GFP_KERNEL_ACCOUNT);
586 static void vgic_its_invalidate_cache(struct vgic_its *its)
588 struct kvm *kvm = its->dev->kvm;
592 xa_for_each(&its->translation_cache, idx, irq) {
593 xa_erase(&its->translation_cache, idx);
601 struct vgic_its *its;
609 its = dev->private;
610 vgic_its_invalidate_cache(its);
616 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
622 if (!its->enabled)
625 ite = find_ite(its, devid, eventid);
636 vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
664 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
671 err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);
713 struct vgic_its *its;
719 its = vgic_msi_to_its(kvm, msi);
720 if (IS_ERR(its))
721 return PTR_ERR(its);
723 mutex_lock(&its->its_lock);
724 ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);
725 mutex_unlock(&its->its_lock);
776 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
783 ite = find_ite(its, device_id, event_id);
790 vgic_its_invalidate_cache(its);
803 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
813 ite = find_ite(its, device_id, event_id);
820 collection = find_collection(its, coll_id);
827 vgic_its_invalidate_cache(its);
832 static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa)
838 idx = srcu_read_lock(&its->dev->kvm->srcu);
839 ret = kvm_is_visible_gfn(its->dev->kvm, gfn);
840 srcu_read_unlock(&its->dev->kvm->srcu, idx);
851 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
885 return __is_visible_gfn_locked(its, addr);
894 if (kvm_read_guest_lock(its->dev->kvm,
915 return __is_visible_gfn_locked(its, indirect_ptr);
922 static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *device,
925 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
934 return __is_visible_gfn_locked(its, gpa);
941 static int vgic_its_alloc_collection(struct vgic_its *its,
954 list_add_tail(&collection->coll_list, &its->collection_list);
960 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
971 collection = find_collection(its, coll_id);
975 for_each_lpi_its(device, ite, its)
1006 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
1019 device = find_its_device(its, device_id);
1023 if (!vgic_its_check_event_id(its, device, event_id))
1035 if (find_ite(its, device_id, event_id))
1038 collection = find_collection(its, coll_id);
1042 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
1045 ret = vgic_its_alloc_collection(its, &collection, coll_id);
1054 vgic_its_free_collection(its, coll_id);
1064 vgic_its_free_collection(its, coll_id);
1074 static void vgic_its_free_device(struct kvm *kvm, struct vgic_its *its,
1087 vgic_its_invalidate_cache(its);
1093 /* its lock must be held */
1094 static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
1098 list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
1099 vgic_its_free_device(kvm, its, cur);
1102 /* its lock must be held */
1103 static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
1107 list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
1108 vgic_its_free_collection(its, cur->collection_id);
1112 static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
1127 list_add_tail(&device->dev_list, &its->device_list);
1135 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
1144 if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))
1150 device = find_its_device(its, device_id);
1158 vgic_its_free_device(kvm, its, device);
1167 device = vgic_its_alloc_device(its, device_id, itt_addr,
1177 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1188 vgic_its_free_collection(its, coll_id);
1189 vgic_its_invalidate_cache(its);
1197 collection = find_collection(its, coll_id);
1202 if (!vgic_its_check_id(its, its->baser_coll_table,
1206 ret = vgic_its_alloc_collection(its, &collection,
1213 update_affinity_collection(kvm, its, collection);
1224 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1232 ite = find_ite(its, device_id, event_id);
1254 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1262 ite = find_ite(its, device_id, event_id);
1307 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1314 collection = find_collection(its, coll_id);
1332 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1360 vgic_its_invalidate_cache(its);
1369 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1375 return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1382 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1387 mutex_lock(&its->its_lock);
1390 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1393 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1396 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1399 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1402 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1405 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1408 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1411 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1414 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1417 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1420 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1427 mutex_unlock(&its->its_lock);
1469 struct vgic_its *its,
1472 return extract_bytes(its->cbaser, addr & 7, len);
1475 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1480 if (its->enabled)
1483 mutex_lock(&its->cmd_lock);
1484 its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1485 its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1486 its->creadr = 0;
1491 its->cwriter = its->creadr;
1492 mutex_unlock(&its->cmd_lock);
1500 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1506 if (!its->enabled)
1509 cbaser = GITS_CBASER_ADDRESS(its->cbaser);
1511 while (its->cwriter != its->creadr) {
1512 int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,
1522 vgic_its_handle_command(kvm, its, cmd_buf);
1524 its->creadr += ITS_CMD_SIZE;
1525 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1526 its->creadr = 0;
1536 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1542 if (!its)
1545 mutex_lock(&its->cmd_lock);
1547 reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1549 if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1550 mutex_unlock(&its->cmd_lock);
1553 its->cwriter = reg;
1555 vgic_its_process_commands(kvm, its);
1557 mutex_unlock(&its->cmd_lock);
1561 struct vgic_its *its,
1564 return extract_bytes(its->cwriter, addr & 0x7, len);
1568 struct vgic_its *its,
1571 return extract_bytes(its->creadr, addr & 0x7, len);
1575 struct vgic_its *its,
1582 mutex_lock(&its->cmd_lock);
1584 if (its->enabled) {
1590 if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1595 its->creadr = cmd_offset;
1597 mutex_unlock(&its->cmd_lock);
1603 struct vgic_its *its,
1610 reg = its->baser_device_table;
1613 reg = its->baser_coll_table;
1625 struct vgic_its *its,
1629 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1634 if (its->enabled)
1639 regptr = &its->baser_device_table;
1644 regptr = &its->baser_coll_table;
1665 mutex_lock(&its->its_lock);
1668 vgic_its_free_device_list(kvm, its);
1671 vgic_its_free_collection_list(kvm, its);
1674 mutex_unlock(&its->its_lock);
1679 struct vgic_its *its,
1684 mutex_lock(&its->cmd_lock);
1685 if (its->creadr == its->cwriter)
1687 if (its->enabled)
1689 mutex_unlock(&its->cmd_lock);
1694 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1698 mutex_lock(&its->cmd_lock);
1704 if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1705 (!(its->baser_device_table & GITS_BASER_VALID) ||
1706 !(its->baser_coll_table & GITS_BASER_VALID) ||
1707 !(its->cbaser & GITS_CBASER_VALID)))
1710 its->enabled = !!(val & GITS_CTLR_ENABLE);
1711 if (!its->enabled)
1712 vgic_its_invalidate_cache(its);
1718 vgic_its_process_commands(kvm, its);
1721 mutex_unlock(&its->cmd_lock);
1743 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1785 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1788 struct vgic_io_device *iodev = &its->iodev;
1792 if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1797 its->vgic_its_base = addr;
1802 iodev->base_addr = its->vgic_its_base;
1804 iodev->its = its;
1827 struct vgic_its *its;
1832 its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL_ACCOUNT);
1833 if (!its)
1842 kfree(its);
1847 mutex_init(&its->its_lock);
1848 mutex_init(&its->cmd_lock);
1852 mutex_lock(&its->cmd_lock);
1853 mutex_lock(&its->its_lock);
1854 mutex_unlock(&its->its_lock);
1855 mutex_unlock(&its->cmd_lock);
1858 its->vgic_its_base = VGIC_ADDR_UNDEF;
1860 INIT_LIST_HEAD(&its->device_list);
1861 INIT_LIST_HEAD(&its->collection_list);
1862 xa_init(&its->translation_cache);
1866 its->enabled = false;
1867 its->dev = dev;
1869 its->baser_device_table = INITIAL_BASER_VALUE |
1871 its->baser_coll_table = INITIAL_BASER_VALUE |
1875 dev->private = its;
1877 ret = vgic_its_set_abi(its, NR_ITS_ABIS - 1);
1887 struct vgic_its *its = kvm_dev->private;
1889 mutex_lock(&its->its_lock);
1891 vgic_its_free_device_list(kvm, its);
1892 vgic_its_free_collection_list(kvm, its);
1893 vgic_its_invalidate_cache(its);
1894 xa_destroy(&its->translation_cache);
1896 mutex_unlock(&its->its_lock);
1897 kfree(its);
1927 struct vgic_its *its;
1932 its = dev->private;
1959 if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1972 addr = its->vgic_its_base + offset;
1978 ret = region->uaccess_its_write(dev->kvm, its, addr,
1981 region->its_write(dev->kvm, its, addr, len, *reg);
1983 *reg = region->its_read(dev->kvm, its, addr, len);
2021 * @its: its handle
2029 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
2036 * @its: its handle
2047 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, u32 esz,
2050 struct kvm *kvm = its->dev->kvm;
2067 next_offset = fn(its, id, entry, opaque);
2085 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
2088 struct kvm *kvm = its->dev->kvm;
2106 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
2111 struct kvm *kvm = its->dev->kvm;
2137 collection = find_collection(its, coll_id);
2141 if (!vgic_its_check_event_id(its, dev, event_id))
2173 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
2175 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2195 ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);
2205 * @its: its handle
2210 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
2212 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2218 ret = scan_its_table(its, base, max_size, ite_esz, 0,
2231 * @its: ITS handle
2235 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
2238 struct kvm *kvm = its->dev->kvm;
2243 next_offset = compute_next_devid_offset(&its->device_list, dev);
2255 * @its: its handle
2263 static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2267 u64 baser = its->baser_device_table;
2288 if (!vgic_its_check_id(its, baser, id, NULL))
2291 dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2295 ret = vgic_its_restore_itt(its, dev);
2297 vgic_its_free_device(its->dev->kvm, its, dev);
2323 static int vgic_its_save_device_tables(struct vgic_its *its)
2325 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2326 u64 baser = its->baser_device_table;
2333 list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2335 list_for_each_entry(dev, &its->device_list, dev_list) {
2339 if (!vgic_its_check_id(its, baser,
2343 ret = vgic_its_save_itt(its, dev);
2347 ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);
2357 * @its: its handle
2366 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2369 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2383 ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2393 static int vgic_its_restore_device_tables(struct vgic_its *its)
2395 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2396 u64 baser = its->baser_device_table;
2408 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2412 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2421 vgic_its_free_device_list(its->dev->kvm, its);
2426 static int vgic_its_save_cte(struct vgic_its *its,
2436 return vgic_write_guest_lock(its->dev->kvm, gpa, &val, esz);
2444 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2447 struct kvm *kvm = its->dev->kvm;
2467 collection = find_collection(its, coll_id);
2471 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
2474 ret = vgic_its_alloc_collection(its, &collection, coll_id);
2485 static int vgic_its_save_collection_table(struct vgic_its *its)
2487 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2488 u64 baser = its->baser_coll_table;
2500 list_for_each_entry(collection, &its->collection_list, coll_list) {
2501 ret = vgic_its_save_cte(its, collection, gpa, cte_esz);
2517 ret = vgic_write_guest_lock(its->dev->kvm, gpa, &val, cte_esz);
2526 static int vgic_its_restore_collection_table(struct vgic_its *its)
2528 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2529 u64 baser = its->baser_coll_table;
2543 ret = vgic_its_restore_cte(its, gpa, cte_esz);
2554 vgic_its_free_collection_list(its->dev->kvm, its);
2563 static int vgic_its_save_tables_v0(struct vgic_its *its)
2567 ret = vgic_its_save_device_tables(its);
2571 return vgic_its_save_collection_table(its);
2579 static int vgic_its_restore_tables_v0(struct vgic_its *its)
2583 ret = vgic_its_restore_collection_table(its);
2587 ret = vgic_its_restore_device_tables(its);
2589 vgic_its_free_collection_list(its->dev->kvm, its);
2593 static int vgic_its_commit_v0(struct vgic_its *its)
2597 abi = vgic_its_get_abi(its);
2598 its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2599 its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2601 its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2604 its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2609 static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2612 its->baser_coll_table &= ~GITS_BASER_VALID;
2613 its->baser_device_table &= ~GITS_BASER_VALID;
2614 its->cbaser = 0;
2615 its->creadr = 0;
2616 its->cwriter = 0;
2617 its->enabled = 0;
2618 vgic_its_free_device_list(kvm, its);
2619 vgic_its_free_collection_list(kvm, its);
2650 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2652 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2666 mutex_lock(&its->its_lock);
2670 vgic_its_reset(kvm, its);
2673 ret = abi->save_tables(its);
2676 ret = abi->restore_tables(its);
2680 mutex_unlock(&its->its_lock);
2693 * dirty guest pages. When vgic/its tables are being saved, the backup
2707 struct vgic_its *its = dev->private;
2722 ret = vgic_check_iorange(dev->kvm, its->vgic_its_base,
2727 return vgic_register_its_iodev(dev->kvm, its, addr);
2730 return vgic_its_ctrl(dev->kvm, its, attr->attr);
2749 struct vgic_its *its = dev->private;
2750 u64 addr = its->vgic_its_base;
2779 .name = "kvm-arm-vgic-its",