Lines Matching defs:its
26 static int vgic_its_save_tables_v0(struct vgic_its *its);
27 static int vgic_its_restore_tables_v0(struct vgic_its *its);
28 static int vgic_its_commit_v0(struct vgic_its *its);
34 * If this LPI is already mapped on another ITS, we increase its refcount
176 int (*save_tables)(struct vgic_its *its);
177 int (*restore_tables)(struct vgic_its *its);
178 int (*commit)(struct vgic_its *its);
197 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
199 return &its_table_abi_versions[its->abi_rev];
202 static int vgic_its_set_abi(struct vgic_its *its, u32 rev)
206 its->abi_rev = rev;
207 abi = vgic_its_get_abi(its);
208 return abi->commit(its);
215 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
219 list_for_each_entry(device, &its->device_list, dev_list)
231 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
237 device = find_its_device(its, device_id);
249 #define for_each_lpi_its(dev, ite, its) \
250 list_for_each_entry(dev, &(its)->device_list, dev_list) \
264 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
268 list_for_each_entry(collection, &its->collection_list, coll_list) {
420 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
426 for_each_lpi_its(device, ite, its) {
498 struct vgic_its *its,
501 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
520 struct vgic_its *its,
525 val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
531 struct vgic_its *its,
539 return vgic_its_set_abi(its, rev);
543 struct vgic_its *its,
618 static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,
641 db = its->vgic_its_base + GITS_TRANSLATER;
661 lockdep_assert_held(&its->its_lock);
699 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
705 if (!its->enabled)
708 ite = find_ite(its, devid, eventid);
719 vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
750 return iodev->its;
760 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
767 err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);
809 struct vgic_its *its;
815 its = vgic_msi_to_its(kvm, msi);
816 if (IS_ERR(its))
817 return PTR_ERR(its);
819 mutex_lock(&its->its_lock);
820 ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);
821 mutex_unlock(&its->its_lock);
872 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
879 ite = find_ite(its, device_id, event_id);
899 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
909 ite = find_ite(its, device_id, event_id);
916 collection = find_collection(its, coll_id);
928 static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa)
934 idx = srcu_read_lock(&its->dev->kvm->srcu);
935 ret = kvm_is_visible_gfn(its->dev->kvm, gfn);
936 srcu_read_unlock(&its->dev->kvm->srcu, idx);
947 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
981 return __is_visible_gfn_locked(its, addr);
990 if (kvm_read_guest_lock(its->dev->kvm,
1011 return __is_visible_gfn_locked(its, indirect_ptr);
1018 static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *device,
1021 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1030 return __is_visible_gfn_locked(its, gpa);
1037 static int vgic_its_alloc_collection(struct vgic_its *its,
1050 list_add_tail(&collection->coll_list, &its->collection_list);
1056 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
1067 collection = find_collection(its, coll_id);
1071 for_each_lpi_its(device, ite, its)
1102 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
1115 device = find_its_device(its, device_id);
1119 if (!vgic_its_check_event_id(its, device, event_id))
1131 if (find_ite(its, device_id, event_id))
1134 collection = find_collection(its, coll_id);
1138 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
1141 ret = vgic_its_alloc_collection(its, &collection, coll_id);
1150 vgic_its_free_collection(its, coll_id);
1160 vgic_its_free_collection(its, coll_id);
1188 /* its lock must be held */
1189 static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
1193 list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
1197 /* its lock must be held */
1198 static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
1202 list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
1203 vgic_its_free_collection(its, cur->collection_id);
1207 static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
1222 list_add_tail(&device->dev_list, &its->device_list);
1230 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
1239 if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))
1245 device = find_its_device(its, device_id);
1262 device = vgic_its_alloc_device(its, device_id, itt_addr,
1272 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1283 vgic_its_free_collection(its, coll_id);
1292 collection = find_collection(its, coll_id);
1297 if (!vgic_its_check_id(its, its->baser_coll_table,
1301 ret = vgic_its_alloc_collection(its, &collection,
1308 update_affinity_collection(kvm, its, collection);
1319 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1327 ite = find_ite(its, device_id, event_id);
1349 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1357 ite = find_ite(its, device_id, event_id);
1406 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1413 collection = find_collection(its, coll_id);
1431 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1473 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1479 return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1486 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1491 mutex_lock(&its->its_lock);
1494 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1497 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1500 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1503 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1506 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1509 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1512 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1515 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1518 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1521 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1524 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1531 mutex_unlock(&its->its_lock);
1573 struct vgic_its *its,
1576 return extract_bytes(its->cbaser, addr & 7, len);
1579 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1584 if (its->enabled)
1587 mutex_lock(&its->cmd_lock);
1588 its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1589 its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1590 its->creadr = 0;
1595 its->cwriter = its->creadr;
1596 mutex_unlock(&its->cmd_lock);
1604 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1610 if (!its->enabled)
1613 cbaser = GITS_CBASER_ADDRESS(its->cbaser);
1615 while (its->cwriter != its->creadr) {
1616 int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,
1626 vgic_its_handle_command(kvm, its, cmd_buf);
1628 its->creadr += ITS_CMD_SIZE;
1629 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1630 its->creadr = 0;
1640 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1646 if (!its)
1649 mutex_lock(&its->cmd_lock);
1651 reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1653 if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1654 mutex_unlock(&its->cmd_lock);
1657 its->cwriter = reg;
1659 vgic_its_process_commands(kvm, its);
1661 mutex_unlock(&its->cmd_lock);
1665 struct vgic_its *its,
1668 return extract_bytes(its->cwriter, addr & 0x7, len);
1672 struct vgic_its *its,
1675 return extract_bytes(its->creadr, addr & 0x7, len);
1679 struct vgic_its *its,
1686 mutex_lock(&its->cmd_lock);
1688 if (its->enabled) {
1694 if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1699 its->creadr = cmd_offset;
1701 mutex_unlock(&its->cmd_lock);
1707 struct vgic_its *its,
1714 reg = its->baser_device_table;
1717 reg = its->baser_coll_table;
1729 struct vgic_its *its,
1733 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1738 if (its->enabled)
1743 regptr = &its->baser_device_table;
1748 regptr = &its->baser_coll_table;
1769 mutex_lock(&its->its_lock);
1772 vgic_its_free_device_list(kvm, its);
1775 vgic_its_free_collection_list(kvm, its);
1778 mutex_unlock(&its->its_lock);
1783 struct vgic_its *its,
1788 mutex_lock(&its->cmd_lock);
1789 if (its->creadr == its->cwriter)
1791 if (its->enabled)
1793 mutex_unlock(&its->cmd_lock);
1798 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1802 mutex_lock(&its->cmd_lock);
1808 if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1809 (!(its->baser_device_table & GITS_BASER_VALID) ||
1810 !(its->baser_coll_table & GITS_BASER_VALID) ||
1811 !(its->cbaser & GITS_CBASER_VALID)))
1814 its->enabled = !!(val & GITS_CTLR_ENABLE);
1815 if (!its->enabled)
1822 vgic_its_process_commands(kvm, its);
1825 mutex_unlock(&its->cmd_lock);
1847 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1889 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1892 struct vgic_io_device *iodev = &its->iodev;
1896 if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1901 its->vgic_its_base = addr;
1906 iodev->base_addr = its->vgic_its_base;
1908 iodev->its = its;
1972 struct vgic_its *its;
1977 its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL_ACCOUNT);
1978 if (!its)
1987 kfree(its);
1994 mutex_init(&its->its_lock);
1995 mutex_init(&its->cmd_lock);
1999 mutex_lock(&its->cmd_lock);
2000 mutex_lock(&its->its_lock);
2001 mutex_unlock(&its->its_lock);
2002 mutex_unlock(&its->cmd_lock);
2005 its->vgic_its_base = VGIC_ADDR_UNDEF;
2007 INIT_LIST_HEAD(&its->device_list);
2008 INIT_LIST_HEAD(&its->collection_list);
2012 its->enabled = false;
2013 its->dev = dev;
2015 its->baser_device_table = INITIAL_BASER_VALUE |
2017 its->baser_coll_table = INITIAL_BASER_VALUE |
2021 dev->private = its;
2023 ret = vgic_its_set_abi(its, NR_ITS_ABIS - 1);
2033 struct vgic_its *its = kvm_dev->private;
2035 mutex_lock(&its->its_lock);
2037 vgic_its_free_device_list(kvm, its);
2038 vgic_its_free_collection_list(kvm, its);
2040 mutex_unlock(&its->its_lock);
2041 kfree(its);
2071 struct vgic_its *its;
2076 its = dev->private;
2103 if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
2116 addr = its->vgic_its_base + offset;
2122 ret = region->uaccess_its_write(dev->kvm, its, addr,
2125 region->its_write(dev->kvm, its, addr, len, *reg);
2127 *reg = region->its_read(dev->kvm, its, addr, len);
2165 * @its: its handle
2173 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
2180 * @its: its handle
2191 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, u32 esz,
2194 struct kvm *kvm = its->dev->kvm;
2211 next_offset = fn(its, id, entry, opaque);
2229 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
2232 struct kvm *kvm = its->dev->kvm;
2250 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
2255 struct kvm *kvm = its->dev->kvm;
2281 collection = find_collection(its, coll_id);
2285 if (!vgic_its_check_event_id(its, dev, event_id))
2317 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
2319 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2339 ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);
2349 * @its: its handle
2354 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
2356 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2362 ret = scan_its_table(its, base, max_size, ite_esz, 0,
2375 * @its: ITS handle
2379 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
2382 struct kvm *kvm = its->dev->kvm;
2387 next_offset = compute_next_devid_offset(&its->device_list, dev);
2399 * @its: its handle
2407 static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2411 u64 baser = its->baser_device_table;
2432 if (!vgic_its_check_id(its, baser, id, NULL))
2435 dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2439 ret = vgic_its_restore_itt(its, dev);
2441 vgic_its_free_device(its->dev->kvm, dev);
2467 static int vgic_its_save_device_tables(struct vgic_its *its)
2469 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2470 u64 baser = its->baser_device_table;
2477 list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2479 list_for_each_entry(dev, &its->device_list, dev_list) {
2483 if (!vgic_its_check_id(its, baser,
2487 ret = vgic_its_save_itt(its, dev);
2491 ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);
2501 * @its: its handle
2510 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2513 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2527 ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2537 static int vgic_its_restore_device_tables(struct vgic_its *its)
2539 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2540 u64 baser = its->baser_device_table;
2552 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2556 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2565 vgic_its_free_device_list(its->dev->kvm, its);
2570 static int vgic_its_save_cte(struct vgic_its *its,
2580 return vgic_write_guest_lock(its->dev->kvm, gpa, &val, esz);
2588 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2591 struct kvm *kvm = its->dev->kvm;
2611 collection = find_collection(its, coll_id);
2615 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
2618 ret = vgic_its_alloc_collection(its, &collection, coll_id);
2629 static int vgic_its_save_collection_table(struct vgic_its *its)
2631 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2632 u64 baser = its->baser_coll_table;
2644 list_for_each_entry(collection, &its->collection_list, coll_list) {
2645 ret = vgic_its_save_cte(its, collection, gpa, cte_esz);
2661 ret = vgic_write_guest_lock(its->dev->kvm, gpa, &val, cte_esz);
2670 static int vgic_its_restore_collection_table(struct vgic_its *its)
2672 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2673 u64 baser = its->baser_coll_table;
2687 ret = vgic_its_restore_cte(its, gpa, cte_esz);
2698 vgic_its_free_collection_list(its->dev->kvm, its);
2707 static int vgic_its_save_tables_v0(struct vgic_its *its)
2711 ret = vgic_its_save_device_tables(its);
2715 return vgic_its_save_collection_table(its);
2723 static int vgic_its_restore_tables_v0(struct vgic_its *its)
2727 ret = vgic_its_restore_collection_table(its);
2731 ret = vgic_its_restore_device_tables(its);
2733 vgic_its_free_collection_list(its->dev->kvm, its);
2737 static int vgic_its_commit_v0(struct vgic_its *its)
2741 abi = vgic_its_get_abi(its);
2742 its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2743 its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2745 its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2748 its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2753 static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2756 its->baser_coll_table &= ~GITS_BASER_VALID;
2757 its->baser_device_table &= ~GITS_BASER_VALID;
2758 its->cbaser = 0;
2759 its->creadr = 0;
2760 its->cwriter = 0;
2761 its->enabled = 0;
2762 vgic_its_free_device_list(kvm, its);
2763 vgic_its_free_collection_list(kvm, its);
2794 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2796 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2810 mutex_lock(&its->its_lock);
2814 vgic_its_reset(kvm, its);
2817 ret = abi->save_tables(its);
2820 ret = abi->restore_tables(its);
2824 mutex_unlock(&its->its_lock);
2837 * dirty guest pages. When vgic/its tables are being saved, the backup
2851 struct vgic_its *its = dev->private;
2866 ret = vgic_check_iorange(dev->kvm, its->vgic_its_base,
2871 return vgic_register_its_iodev(dev->kvm, its, addr);
2874 return vgic_its_ctrl(dev->kvm, its, attr->attr);
2893 struct vgic_its *its = dev->private;
2894 u64 addr = its->vgic_its_base;
2923 .name = "kvm-arm-vgic-its",