hw/mlx5/odp.c

100 			struct mlx5_ib_mr *imr, int flags)
107 			pklm->key = mr_to_mdev(imr)->mkeys.null_mkey;
130 	lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex);
133 		struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx);
140 			pklm->key = mr_to_mdev(imr)->mkeys.null_mkey;
194 	struct mlx5_ib_mr *imr = mr->parent;
195 	struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem);
207 	mlx5r_deref_odp_mkey(&imr->mmkey);
214 	struct mlx5_ib_mr *imr = mr->parent;
216 	if (!refcount_inc_not_zero(&imr->mmkey.usecount))
219 	xa_erase(&imr->implicit_children, idx);
405 static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr,
408 	struct mlx5_ib_dev *dev = mr_to_mdev(imr);
414 	odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem),
420 	mr = mlx5_mr_cache_alloc(dev, imr->access_flags,
428 	mr->access_flags = imr->access_flags;
429 	mr->ibmr.pd = imr->ibmr.pd;
430 	mr->ibmr.device = &mr_to_mdev(imr)->ib_dev;
435 	mr->parent = imr;
454 	xa_lock(&imr->implicit_children);
455 	ret = __xa_cmpxchg(&imr->implicit_children, idx, NULL, mr,
469 	xa_unlock(&imr->implicit_children);
471 	mlx5_ib_dbg(mr_to_mdev(imr), "key %x mr %p\n", mr->mmkey.key, mr);
475 	xa_unlock(&imr->implicit_children);
486 	struct mlx5_ib_mr *imr;
496 	imr = mlx5_mr_cache_alloc(dev, access_flags, MLX5_MKC_ACCESS_MODE_KSM,
498 	if (IS_ERR(imr)) {
500 		return imr;
503 	imr->access_flags = access_flags;
504 	imr->ibmr.pd = &pd->ibpd;
505 	imr->ibmr.iova = 0;
506 	imr->umem = &umem_odp->umem;
507 	imr->ibmr.lkey = imr->mmkey.key;
508 	imr->ibmr.rkey = imr->mmkey.key;
509 	imr->ibmr.device = &dev->ib_dev;
510 	imr->is_odp_implicit = true;
511 	xa_init(&imr->implicit_children);
513 	err = mlx5r_umr_update_xlt(imr, 0,
522 	err = mlx5r_store_odp_mkey(dev, &imr->mmkey);
526 	mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr);
527 	return imr;
530 	mlx5_ib_dereg_mr(&imr->ibmr, NULL);
604 static int pagefault_implicit_mr(struct mlx5_ib_mr *imr,
626 		xa_lock(&imr->implicit_children);
627 		mtt = xa_load(&imr->implicit_children, idx);
629 			xa_unlock(&imr->implicit_children);
630 			mtt = implicit_get_child_mr(imr, idx);
639 			xa_unlock(&imr->implicit_children);
679 	err = mlx5r_umr_update_xlt(imr, upd_start_idx, upd_len, 0,
684 		mlx5_ib_err(mr_to_mdev(imr), "Failed to update PAS\n");