Lines Matching refs:mm
549 static void _ppgtt_get_root_entry(struct intel_vgpu_mm *mm,
553 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
555 GEM_BUG_ON(mm->type != INTEL_GVT_MM_PPGTT);
557 entry->type = mm->ppgtt_mm.root_entry_type;
558 pte_ops->get_entry(guest ? mm->ppgtt_mm.guest_pdps :
559 mm->ppgtt_mm.shadow_pdps,
560 entry, index, false, 0, mm->vgpu);
564 static inline void ppgtt_get_guest_root_entry(struct intel_vgpu_mm *mm,
567 _ppgtt_get_root_entry(mm, entry, index, true);
570 static inline void ppgtt_get_shadow_root_entry(struct intel_vgpu_mm *mm,
573 _ppgtt_get_root_entry(mm, entry, index, false);
576 static void _ppgtt_set_root_entry(struct intel_vgpu_mm *mm,
580 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
582 pte_ops->set_entry(guest ? mm->ppgtt_mm.guest_pdps :
583 mm->ppgtt_mm.shadow_pdps,
584 entry, index, false, 0, mm->vgpu);
587 static inline void ppgtt_set_shadow_root_entry(struct intel_vgpu_mm *mm,
590 _ppgtt_set_root_entry(mm, entry, index, false);
593 static void ggtt_get_guest_entry(struct intel_vgpu_mm *mm,
596 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
598 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
601 pte_ops->get_entry(mm->ggtt_mm.virtual_ggtt, entry, index,
602 false, 0, mm->vgpu);
605 static void ggtt_set_guest_entry(struct intel_vgpu_mm *mm,
608 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
610 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
612 pte_ops->set_entry(mm->ggtt_mm.virtual_ggtt, entry, index,
613 false, 0, mm->vgpu);
616 static void ggtt_get_host_entry(struct intel_vgpu_mm *mm,
619 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
621 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
623 pte_ops->get_entry(NULL, entry, index, false, 0, mm->vgpu);
626 static void ggtt_set_host_entry(struct intel_vgpu_mm *mm,
629 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
632 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
634 if (vgpu_gmadr_is_aperture(mm->vgpu, index << I915_GTT_PAGE_SHIFT)) {
635 offset -= (vgpu_aperture_gmadr_base(mm->vgpu) >> PAGE_SHIFT);
636 mm->ggtt_mm.host_ggtt_aperture[offset] = entry->val64;
637 } else if (vgpu_gmadr_is_hidden(mm->vgpu, index << I915_GTT_PAGE_SHIFT)) {
638 offset -= (vgpu_hidden_gmadr_base(mm->vgpu) >> PAGE_SHIFT);
639 mm->ggtt_mm.host_ggtt_hidden[offset] = entry->val64;
642 pte_ops->set_entry(NULL, entry, index, false, 0, mm->vgpu);
1753 static void invalidate_ppgtt_mm(struct intel_vgpu_mm *mm)
1755 struct intel_vgpu *vgpu = mm->vgpu;
1762 if (!mm->ppgtt_mm.shadowed)
1765 for (index = 0; index < ARRAY_SIZE(mm->ppgtt_mm.shadow_pdps); index++) {
1766 ppgtt_get_shadow_root_entry(mm, &se, index);
1773 ppgtt_set_shadow_root_entry(mm, &se, index);
1779 mm->ppgtt_mm.shadowed = false;
1783 static int shadow_ppgtt_mm(struct intel_vgpu_mm *mm)
1785 struct intel_vgpu *vgpu = mm->vgpu;
1793 if (mm->ppgtt_mm.shadowed)
1799 mm->ppgtt_mm.shadowed = true;
1801 for (index = 0; index < ARRAY_SIZE(mm->ppgtt_mm.guest_pdps); index++) {
1802 ppgtt_get_guest_root_entry(mm, &ge, index);
1817 ppgtt_set_shadow_root_entry(mm, &se, index);
1825 invalidate_ppgtt_mm(mm);
1831 struct intel_vgpu_mm *mm;
1833 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
1834 if (!mm)
1837 mm->vgpu = vgpu;
1838 kref_init(&mm->ref);
1839 atomic_set(&mm->pincount, 0);
1841 return mm;
1844 static void vgpu_free_mm(struct intel_vgpu_mm *mm)
1846 kfree(mm);
1850 * intel_vgpu_create_ppgtt_mm - create a ppgtt mm object for a vGPU
1855 * This function is used to create a ppgtt mm object for a vGPU.
1864 struct intel_vgpu_mm *mm;
1867 mm = vgpu_alloc_mm(vgpu);
1868 if (!mm)
1871 mm->type = INTEL_GVT_MM_PPGTT;
1875 mm->ppgtt_mm.root_entry_type = root_entry_type;
1877 INIT_LIST_HEAD(&mm->ppgtt_mm.list);
1878 INIT_LIST_HEAD(&mm->ppgtt_mm.lru_list);
1879 INIT_LIST_HEAD(&mm->ppgtt_mm.link);
1882 mm->ppgtt_mm.guest_pdps[0] = pdps[0];
1884 memcpy(mm->ppgtt_mm.guest_pdps, pdps,
1885 sizeof(mm->ppgtt_mm.guest_pdps));
1887 ret = shadow_ppgtt_mm(mm);
1889 gvt_vgpu_err("failed to shadow ppgtt mm\n");
1890 vgpu_free_mm(mm);
1894 list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head);
1897 list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head);
1900 return mm;
1905 struct intel_vgpu_mm *mm;
1908 mm = vgpu_alloc_mm(vgpu);
1909 if (!mm)
1912 mm->type = INTEL_GVT_MM_GGTT;
1915 mm->ggtt_mm.virtual_ggtt =
1918 if (!mm->ggtt_mm.virtual_ggtt) {
1919 vgpu_free_mm(mm);
1923 mm->ggtt_mm.host_ggtt_aperture = vzalloc((vgpu_aperture_sz(vgpu) >> PAGE_SHIFT) * sizeof(u64));
1924 if (!mm->ggtt_mm.host_ggtt_aperture) {
1925 vfree(mm->ggtt_mm.virtual_ggtt);
1926 vgpu_free_mm(mm);
1930 mm->ggtt_mm.host_ggtt_hidden = vzalloc((vgpu_hidden_sz(vgpu) >> PAGE_SHIFT) * sizeof(u64));
1931 if (!mm->ggtt_mm.host_ggtt_hidden) {
1932 vfree(mm->ggtt_mm.host_ggtt_aperture);
1933 vfree(mm->ggtt_mm.virtual_ggtt);
1934 vgpu_free_mm(mm);
1938 return mm;
1942 * _intel_vgpu_mm_release - destroy a mm object
1945 * This function is used to destroy a mm object for vGPU
1950 struct intel_vgpu_mm *mm = container_of(mm_ref, typeof(*mm), ref);
1952 if (GEM_WARN_ON(atomic_read(&mm->pincount)))
1953 gvt_err("vgpu mm pin count bug detected\n");
1955 if (mm->type == INTEL_GVT_MM_PPGTT) {
1956 list_del(&mm->ppgtt_mm.list);
1958 mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);
1959 list_del(&mm->ppgtt_mm.lru_list);
1960 mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);
1962 invalidate_ppgtt_mm(mm);
1964 vfree(mm->ggtt_mm.virtual_ggtt);
1965 vfree(mm->ggtt_mm.host_ggtt_aperture);
1966 vfree(mm->ggtt_mm.host_ggtt_hidden);
1969 vgpu_free_mm(mm);
1973 * intel_vgpu_unpin_mm - decrease the pin count of a vGPU mm object
1974 * @mm: a vGPU mm object
1976 * This function is called when user doesn't want to use a vGPU mm object
1978 void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm)
1980 atomic_dec_if_positive(&mm->pincount);
1984 * intel_vgpu_pin_mm - increase the pin count of a vGPU mm object
1985 * @mm: target vgpu mm
1987 * This function is called when user wants to use a vGPU mm object. If this
1988 * mm object hasn't been shadowed yet, the shadow will be populated at this
1994 int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm)
1998 atomic_inc(&mm->pincount);
2000 if (mm->type == INTEL_GVT_MM_PPGTT) {
2001 ret = shadow_ppgtt_mm(mm);
2005 mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);
2006 list_move_tail(&mm->ppgtt_mm.lru_list,
2007 &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head);
2008 mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);
2016 struct intel_vgpu_mm *mm;
2022 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.lru_list);
2024 if (atomic_read(&mm->pincount))
2027 list_del_init(&mm->ppgtt_mm.lru_list);
2029 invalidate_ppgtt_mm(mm);
2039 static inline int ppgtt_get_next_level_entry(struct intel_vgpu_mm *mm,
2042 struct intel_vgpu *vgpu = mm->vgpu;
2059 * @mm: mm object. could be a PPGTT or GGTT mm object
2060 * @gma: graphics memory address in this mm object
2068 unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
2070 struct intel_vgpu *vgpu = mm->vgpu;
2080 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT &&
2081 mm->type != INTEL_GVT_MM_PPGTT);
2083 if (mm->type == INTEL_GVT_MM_GGTT) {
2087 ggtt_get_guest_entry(mm, &e,
2095 switch (mm->ppgtt_mm.root_entry_type) {
2097 ppgtt_get_shadow_root_entry(mm, &e, 0);
2106 ppgtt_get_shadow_root_entry(mm, &e,
2119 ret = ppgtt_get_next_level_entry(mm, &e, gma_index[i],
2133 mm->ppgtt_mm.root_entry_type, gma, gpa);
2138 gvt_vgpu_err("invalid mm type: %d gma %lx\n", mm->type, gma);
2475 gvt_vgpu_err("fail to create mm for ggtt.\n");
2489 struct intel_vgpu_mm *mm;
2492 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list);
2493 intel_vgpu_destroy_mm(mm);
2497 gvt_err("vgpu ppgtt mm is not fully destroyed\n");
2592 * intel_vgpu_find_ppgtt_mm - find a PPGTT mm object
2596 * This function is used to find a PPGTT mm object from mm object pool
2599 * pointer to mm object on success, NULL if failed.
2604 struct intel_vgpu_mm *mm;
2608 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list);
2610 switch (mm->ppgtt_mm.root_entry_type) {
2612 if (pdps[0] == mm->ppgtt_mm.guest_pdps[0])
2613 return mm;
2616 if (!memcmp(pdps, mm->ppgtt_mm.guest_pdps,
2617 sizeof(mm->ppgtt_mm.guest_pdps)))
2618 return mm;
2628 * intel_vgpu_get_ppgtt_mm - get or create a PPGTT mm object.
2633 * This function is used to find or create a PPGTT mm object from a guest.
2641 struct intel_vgpu_mm *mm;
2643 mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps);
2644 if (mm) {
2645 intel_vgpu_mm_get(mm);
2647 mm = intel_vgpu_create_ppgtt_mm(vgpu, root_entry_type, pdps);
2648 if (IS_ERR(mm))
2649 gvt_vgpu_err("fail to create mm\n");
2651 return mm;
2655 * intel_vgpu_put_ppgtt_mm - find and put a PPGTT mm object.
2659 * This function is used to find a PPGTT mm object from a guest and destroy it.
2666 struct intel_vgpu_mm *mm;
2668 mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps);
2669 if (!mm) {
2673 intel_vgpu_mm_put(mm);
2678 * intel_gvt_init_gtt - initialize mm components of a GVT device
2682 * the mm components of a GVT device.
2731 * intel_gvt_clean_gtt - clean up mm components of a GVT device
2735 * the mm components of a GVT device.
2762 struct intel_vgpu_mm *mm;
2765 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list);
2766 if (mm->type == INTEL_GVT_MM_PPGTT) {
2768 list_del_init(&mm->ppgtt_mm.lru_list);
2770 if (mm->ppgtt_mm.shadowed)
2771 invalidate_ppgtt_mm(mm);
2831 struct intel_vgpu_mm *mm;
2838 mm = vgpu->gtt.ggtt_mm;
2843 pte = mm->ggtt_mm.host_ggtt_aperture[idx];
2851 pte = mm->ggtt_mm.host_ggtt_hidden[idx];