1322810Shselasky/*- 2322810Shselasky * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3322810Shselasky * 4322810Shselasky * Redistribution and use in source and binary forms, with or without 5322810Shselasky * modification, are permitted provided that the following conditions 6322810Shselasky * are met: 7322810Shselasky * 1. Redistributions of source code must retain the above copyright 8322810Shselasky * notice, this list of conditions and the following disclaimer. 9322810Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10322810Shselasky * notice, this list of conditions and the following disclaimer in the 11322810Shselasky * documentation and/or other materials provided with the distribution. 12322810Shselasky * 13322810Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14322810Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15322810Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16322810Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17322810Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18322810Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19322810Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20322810Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21322810Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22322810Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23322810Shselasky * SUCH DAMAGE. 24322810Shselasky * 25322810Shselasky * $FreeBSD: stable/10/sys/dev/mlx5/mlx5_ib/mlx5_ib_mr.c 323223 2017-09-06 15:33:23Z hselasky $ 26322810Shselasky */ 27322810Shselasky 28323223Shselasky#include <linux/compiler.h> 29322810Shselasky#include <linux/kref.h> 30322810Shselasky#include <linux/random.h> 31322810Shselasky#include <linux/fs.h> 32322810Shselasky#include <linux/delay.h> 33322810Shselasky#include <rdma/ib_umem.h> 34322810Shselasky#include "mlx5_ib.h" 35322810Shselasky 36322810ShselaskyCTASSERT((uintptr_t)PAGE_MASK > (uintptr_t)PAGE_SIZE); 37322810Shselasky 38322810Shselaskyenum { 39322810Shselasky MAX_PENDING_REG_MR = 8, 40322810Shselasky MAX_MR_RELEASE_TIMEOUT = (60 * 20) /* Allow release timeout up to 20 min */ 41322810Shselasky}; 42322810Shselasky 43322810Shselasky#define MLX5_UMR_ALIGN 2048 44322810Shselasky 45322810Shselaskystatic int mlx5_mr_sysfs_init(struct mlx5_ib_dev *dev); 46322810Shselaskystatic void mlx5_mr_sysfs_cleanup(struct mlx5_ib_dev *dev); 47322810Shselasky 48322810Shselaskystatic int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 49322810Shselasky{ 50322810Shselasky int err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr); 51322810Shselasky 52322810Shselasky return err; 53322810Shselasky} 54322810Shselasky 55322810Shselaskystatic int order2idx(struct mlx5_ib_dev *dev, int order) 56322810Shselasky{ 57322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 58322810Shselasky 59322810Shselasky if (order < cache->ent[0].order) 60322810Shselasky return 0; 61322810Shselasky else 62322810Shselasky return order - cache->ent[0].order; 63322810Shselasky} 64322810Shselasky 65322810Shselaskystatic void reg_mr_callback(int status, void *context) 66322810Shselasky{ 67322810Shselasky struct mlx5_ib_mr *mr = context; 68322810Shselasky struct mlx5_ib_dev *dev = mr->dev; 69322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 70322810Shselasky int c = order2idx(dev, mr->order); 71322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[c]; 72322810Shselasky struct mlx5_core_dev *mdev = dev->mdev; 73322810Shselasky struct mlx5_core_mr *mmr = &mr->mmr; 74322810Shselasky struct mlx5_mr_table *table = &dev->mdev->priv.mr_table; 75322810Shselasky unsigned long flags; 76322810Shselasky int err; 77322810Shselasky u8 key; 78322810Shselasky 79322810Shselasky spin_lock_irqsave(&ent->lock, flags); 80322810Shselasky ent->pending--; 81322810Shselasky spin_unlock_irqrestore(&ent->lock, flags); 82322810Shselasky if (status) { 83322810Shselasky mlx5_ib_warn(dev, "async reg mr failed. status %d, order %d\n", status, ent->order); 84322810Shselasky kfree(mr); 85322810Shselasky dev->fill_delay = 1; 86322810Shselasky mod_timer(&dev->delay_timer, jiffies + HZ); 87322810Shselasky return; 88322810Shselasky } 89322810Shselasky 90322810Shselasky if (mr->out.hdr.status) { 91322810Shselasky mlx5_ib_warn(dev, "failed - status %d, syndorme 0x%x\n", 92322810Shselasky mr->out.hdr.status, 93322810Shselasky be32_to_cpu(mr->out.hdr.syndrome)); 94322810Shselasky kfree(mr); 95322810Shselasky dev->fill_delay = 1; 96322810Shselasky mod_timer(&dev->delay_timer, jiffies + HZ); 97322810Shselasky return; 98322810Shselasky } 99322810Shselasky 100322810Shselasky spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags); 101322810Shselasky key = dev->mdev->priv.mkey_key++; 102322810Shselasky spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags); 103322810Shselasky mmr->key = mlx5_idx_to_mkey(be32_to_cpu(mr->out.mkey) & 0xffffff) | key; 104322810Shselasky mlx5_ib_dbg(dev, "callbacked mkey 0x%x created\n", 105322810Shselasky be32_to_cpu(mr->out.mkey)); 106322810Shselasky 107322810Shselasky cache->last_add = jiffies; 108322810Shselasky 109322810Shselasky spin_lock_irqsave(&ent->lock, flags); 110322810Shselasky list_add_tail(&mr->list, &ent->head); 111322810Shselasky ent->cur++; 112322810Shselasky ent->size++; 113322810Shselasky spin_unlock_irqrestore(&ent->lock, flags); 114322810Shselasky 115322810Shselasky spin_lock_irqsave(&table->lock, flags); 116322810Shselasky err = radix_tree_insert(&table->tree, mlx5_mkey_to_idx(mmr->key), mmr); 117322810Shselasky spin_unlock_irqrestore(&table->lock, flags); 118322810Shselasky if (err) { 119322810Shselasky mlx5_ib_warn(dev, "failed radix tree insert of mkey 0x%x, %d\n", 120322810Shselasky mmr->key, err); 121322810Shselasky mlx5_core_destroy_mkey(mdev, mmr); 122322810Shselasky } 123322810Shselasky} 124322810Shselasky 125322810Shselaskystatic int add_keys(struct mlx5_ib_dev *dev, int c, int num) 126322810Shselasky{ 127322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 128322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[c]; 129322810Shselasky struct mlx5_create_mkey_mbox_in *in; 130322810Shselasky struct mlx5_ib_mr *mr; 131322810Shselasky int npages = 1 << ent->order; 132322810Shselasky int err = 0; 133322810Shselasky int i; 134322810Shselasky 135322810Shselasky in = kzalloc(sizeof(*in), GFP_KERNEL); 136322810Shselasky if (!in) 137322810Shselasky return -ENOMEM; 138322810Shselasky 139322810Shselasky for (i = 0; i < num; i++) { 140322810Shselasky if (ent->pending >= MAX_PENDING_REG_MR) { 141322810Shselasky err = -EAGAIN; 142322810Shselasky break; 143322810Shselasky } 144322810Shselasky 145322810Shselasky mr = kzalloc(sizeof(*mr), GFP_KERNEL); 146322810Shselasky if (!mr) { 147322810Shselasky err = -ENOMEM; 148322810Shselasky break; 149322810Shselasky } 150322810Shselasky mr->order = ent->order; 151322810Shselasky mr->umred = 1; 152322810Shselasky mr->dev = dev; 153322810Shselasky in->seg.status = MLX5_MKEY_STATUS_FREE; 154322810Shselasky in->seg.xlt_oct_size = cpu_to_be32((npages + 1) / 2); 155322810Shselasky in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 156322810Shselasky in->seg.flags = MLX5_ACCESS_MODE_MTT | MLX5_PERM_UMR_EN; 157322810Shselasky in->seg.log2_page_size = 12; 158322810Shselasky 159322810Shselasky spin_lock_irq(&ent->lock); 160322810Shselasky ent->pending++; 161322810Shselasky spin_unlock_irq(&ent->lock); 162322810Shselasky err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, 163322810Shselasky sizeof(*in), reg_mr_callback, 164322810Shselasky mr, &mr->out); 165322810Shselasky if (err) { 166322810Shselasky spin_lock_irq(&ent->lock); 167322810Shselasky ent->pending--; 168322810Shselasky spin_unlock_irq(&ent->lock); 169322810Shselasky mlx5_ib_warn(dev, "create mkey failed %d\n", err); 170322810Shselasky kfree(mr); 171322810Shselasky break; 172322810Shselasky } 173322810Shselasky } 174322810Shselasky 175322810Shselasky kfree(in); 176322810Shselasky return err; 177322810Shselasky} 178322810Shselasky 179322810Shselaskystatic void remove_keys(struct mlx5_ib_dev *dev, int c, int num) 180322810Shselasky{ 181322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 182322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[c]; 183322810Shselasky struct mlx5_ib_mr *mr; 184322810Shselasky int err; 185322810Shselasky int i; 186322810Shselasky 187322810Shselasky for (i = 0; i < num; i++) { 188322810Shselasky spin_lock_irq(&ent->lock); 189322810Shselasky if (list_empty(&ent->head)) { 190322810Shselasky spin_unlock_irq(&ent->lock); 191322810Shselasky return; 192322810Shselasky } 193322810Shselasky mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 194322810Shselasky list_del(&mr->list); 195322810Shselasky ent->cur--; 196322810Shselasky ent->size--; 197322810Shselasky spin_unlock_irq(&ent->lock); 198322810Shselasky err = destroy_mkey(dev, mr); 199322810Shselasky if (err) 200322810Shselasky mlx5_ib_warn(dev, "failed destroy mkey\n"); 201322810Shselasky else 202322810Shselasky kfree(mr); 203322810Shselasky } 204322810Shselasky} 205322810Shselasky 206322810Shselaskystatic int someone_adding(struct mlx5_mr_cache *cache) 207322810Shselasky{ 208322810Shselasky int i; 209322810Shselasky 210322810Shselasky for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 211322810Shselasky if (cache->ent[i].cur < cache->ent[i].limit) 212322810Shselasky return 1; 213322810Shselasky } 214322810Shselasky 215322810Shselasky return 0; 216322810Shselasky} 217322810Shselasky 218322810Shselaskystatic int someone_releasing(struct mlx5_mr_cache *cache) 219322810Shselasky{ 220322810Shselasky int i; 221322810Shselasky 222322810Shselasky for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 223322810Shselasky if (cache->ent[i].cur > 2 * cache->ent[i].limit) 224322810Shselasky return 1; 225322810Shselasky } 226322810Shselasky 227322810Shselasky return 0; 228322810Shselasky} 229322810Shselasky 230322810Shselaskystatic void __cache_work_func(struct mlx5_cache_ent *ent) 231322810Shselasky{ 232322810Shselasky struct mlx5_ib_dev *dev = ent->dev; 233322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 234322810Shselasky int i = order2idx(dev, ent->order); 235322810Shselasky int err; 236322810Shselasky s64 dtime; 237322810Shselasky 238322810Shselasky if (cache->stopped) 239322810Shselasky return; 240322810Shselasky 241322810Shselasky ent = &dev->cache.ent[i]; 242322810Shselasky if (ent->cur < 2 * ent->limit && !dev->fill_delay) { 243322810Shselasky err = add_keys(dev, i, 1); 244322810Shselasky if (ent->cur < 2 * ent->limit) { 245322810Shselasky if (err == -EAGAIN) { 246322810Shselasky mlx5_ib_dbg(dev, "returned eagain, order %d\n", 247322810Shselasky i + 2); 248322810Shselasky cancel_delayed_work(&ent->dwork); 249322810Shselasky if (!queue_delayed_work(cache->wq, &ent->dwork, 250322810Shselasky msecs_to_jiffies(3))) 251322810Shselasky mlx5_ib_warn(dev, "failed queueing delayed work\n"); 252322810Shselasky } else if (err) { 253322810Shselasky mlx5_ib_warn(dev, "command failed order %d, err %d\n", 254322810Shselasky i + 2, err); 255322810Shselasky cancel_delayed_work(&ent->dwork); 256322810Shselasky if (!queue_delayed_work(cache->wq, &ent->dwork, 257322810Shselasky msecs_to_jiffies(1000))) 258322810Shselasky mlx5_ib_warn(dev, "failed queueing delayed work\n"); 259322810Shselasky } else { 260322810Shselasky if (!queue_work(cache->wq, &ent->work)) 261322810Shselasky mlx5_ib_warn(dev, "failed queueing work\n"); 262322810Shselasky } 263322810Shselasky } 264322810Shselasky } else if (ent->cur > 2 * ent->limit) { 265322810Shselasky dtime = (cache->last_add + (s64)cache->rel_timeout * HZ) - jiffies; 266322810Shselasky if (cache->rel_imm || 267322810Shselasky (cache->rel_timeout >= 0 && !someone_adding(cache) && dtime <= 0)) { 268322810Shselasky remove_keys(dev, i, 1); 269322810Shselasky if (ent->cur > ent->limit) 270322810Shselasky if (!queue_work(cache->wq, &ent->work)) 271322810Shselasky mlx5_ib_warn(dev, "failed queueing work\n"); 272322810Shselasky } else if (cache->rel_timeout >= 0) { 273322810Shselasky dtime = max_t(s64, dtime, 0); 274322810Shselasky dtime = min_t(s64, dtime, (MAX_MR_RELEASE_TIMEOUT * HZ)); 275322810Shselasky cancel_delayed_work(&ent->dwork); 276322810Shselasky if (!queue_delayed_work(cache->wq, &ent->dwork, dtime)) 277322810Shselasky mlx5_ib_warn(dev, "failed queueing delayed work\n"); 278322810Shselasky } 279322810Shselasky } else if (cache->rel_imm && !someone_releasing(cache)) { 280322810Shselasky cache->rel_imm = 0; 281322810Shselasky } 282322810Shselasky} 283322810Shselasky 284322810Shselaskystatic void delayed_cache_work_func(struct work_struct *work) 285322810Shselasky{ 286322810Shselasky struct mlx5_cache_ent *ent; 287322810Shselasky 288322810Shselasky ent = container_of(work, struct mlx5_cache_ent, dwork.work); 289322810Shselasky __cache_work_func(ent); 290322810Shselasky} 291322810Shselasky 292322810Shselaskystatic void cache_work_func(struct work_struct *work) 293322810Shselasky{ 294322810Shselasky struct mlx5_cache_ent *ent; 295322810Shselasky 296322810Shselasky ent = container_of(work, struct mlx5_cache_ent, work); 297322810Shselasky __cache_work_func(ent); 298322810Shselasky} 299322810Shselasky 300322810Shselaskystatic void free_cached_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 301322810Shselasky{ 302322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 303322810Shselasky struct mlx5_cache_ent *ent; 304322810Shselasky int shrink = 0; 305322810Shselasky int c; 306322810Shselasky 307322810Shselasky c = order2idx(dev, mr->order); 308322810Shselasky if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) { 309322810Shselasky mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c); 310322810Shselasky return; 311322810Shselasky } 312322810Shselasky ent = &cache->ent[c]; 313322810Shselasky spin_lock_irq(&ent->lock); 314322810Shselasky list_add_tail(&mr->list, &ent->head); 315322810Shselasky ent->cur++; 316322810Shselasky if (ent->cur > 2 * ent->limit) 317322810Shselasky shrink = 1; 318322810Shselasky spin_unlock_irq(&ent->lock); 319322810Shselasky 320322810Shselasky if (shrink) 321322810Shselasky if (!queue_work(cache->wq, &ent->work)) 322322810Shselasky mlx5_ib_warn(dev, "failed queueing work\n"); 323322810Shselasky} 324322810Shselasky 325322810Shselaskystatic void clean_keys(struct mlx5_ib_dev *dev, int c) 326322810Shselasky{ 327322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 328322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[c]; 329322810Shselasky struct mlx5_ib_mr *mr; 330322810Shselasky int err; 331322810Shselasky 332322810Shselasky cancel_delayed_work(&ent->dwork); 333322810Shselasky while (1) { 334322810Shselasky spin_lock_irq(&ent->lock); 335322810Shselasky if (list_empty(&ent->head)) { 336322810Shselasky spin_unlock_irq(&ent->lock); 337322810Shselasky return; 338322810Shselasky } 339322810Shselasky mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 340322810Shselasky list_del(&mr->list); 341322810Shselasky ent->cur--; 342322810Shselasky ent->size--; 343322810Shselasky spin_unlock_irq(&ent->lock); 344322810Shselasky err = destroy_mkey(dev, mr); 345322810Shselasky if (err) 346322810Shselasky mlx5_ib_warn(dev, "failed destroy mkey 0x%x from order %d\n", 347322810Shselasky mr->mmr.key, ent->order); 348322810Shselasky else 349322810Shselasky kfree(mr); 350322810Shselasky } 351322810Shselasky} 352322810Shselasky 353322810Shselaskystatic void delay_time_func(unsigned long ctx) 354322810Shselasky{ 355322810Shselasky struct mlx5_ib_dev *dev = (struct mlx5_ib_dev *)ctx; 356322810Shselasky 357322810Shselasky dev->fill_delay = 0; 358322810Shselasky} 359322810Shselasky 360322810Shselaskyenum { 361322810Shselasky MLX5_VF_MR_LIMIT = 2, 362322810Shselasky}; 363322810Shselasky 364322810Shselaskyint mlx5_mr_cache_init(struct mlx5_ib_dev *dev) 365322810Shselasky{ 366322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 367322810Shselasky struct mlx5_cache_ent *ent; 368322810Shselasky int limit; 369322810Shselasky int err; 370322810Shselasky int i; 371322810Shselasky 372322810Shselasky mutex_init(&dev->slow_path_mutex); 373322810Shselasky cache->rel_timeout = 300; 374322810Shselasky cache->wq = create_singlethread_workqueue("mkey_cache"); 375322810Shselasky if (!cache->wq) { 376322810Shselasky mlx5_ib_warn(dev, "failed to create work queue\n"); 377322810Shselasky return -ENOMEM; 378322810Shselasky } 379322810Shselasky 380322810Shselasky setup_timer(&dev->delay_timer, delay_time_func, (uintptr_t)dev); 381322810Shselasky for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 382322810Shselasky INIT_LIST_HEAD(&cache->ent[i].head); 383322810Shselasky spin_lock_init(&cache->ent[i].lock); 384322810Shselasky 385322810Shselasky ent = &cache->ent[i]; 386322810Shselasky INIT_LIST_HEAD(&ent->head); 387322810Shselasky spin_lock_init(&ent->lock); 388322810Shselasky ent->order = i + 2; 389322810Shselasky ent->dev = dev; 390322810Shselasky 391322810Shselasky if (dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) { 392322810Shselasky if (mlx5_core_is_pf(dev->mdev)) 393322810Shselasky limit = dev->mdev->profile->mr_cache[i].limit; 394322810Shselasky else 395322810Shselasky limit = MLX5_VF_MR_LIMIT; 396322810Shselasky } else { 397322810Shselasky limit = 0; 398322810Shselasky } 399322810Shselasky 400322810Shselasky INIT_WORK(&ent->work, cache_work_func); 401322810Shselasky INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); 402322810Shselasky ent->limit = limit; 403322810Shselasky if (!queue_work(cache->wq, &ent->work)) 404322810Shselasky mlx5_ib_warn(dev, "failed queueing work\n"); 405322810Shselasky } 406322810Shselasky 407322810Shselasky err = mlx5_mr_sysfs_init(dev); 408322810Shselasky if (err) 409322810Shselasky mlx5_ib_warn(dev, "failed to init mr cache sysfs\n"); 410322810Shselasky 411322810Shselasky return 0; 412322810Shselasky} 413322810Shselasky 414322810Shselaskystatic void wait_for_async_commands(struct mlx5_ib_dev *dev) 415322810Shselasky{ 416322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 417322810Shselasky struct mlx5_cache_ent *ent; 418322810Shselasky int total = 0; 419322810Shselasky int i; 420322810Shselasky int j; 421322810Shselasky 422322810Shselasky for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 423322810Shselasky ent = &cache->ent[i]; 424322810Shselasky for (j = 0 ; j < 1000; j++) { 425322810Shselasky if (!ent->pending) 426322810Shselasky break; 427322810Shselasky msleep(50); 428322810Shselasky } 429322810Shselasky } 430322810Shselasky for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 431322810Shselasky ent = &cache->ent[i]; 432322810Shselasky total += ent->pending; 433322810Shselasky } 434322810Shselasky 435322810Shselasky if (total) 436322810Shselasky mlx5_ib_dbg(dev, "aborted, %d pending requests\n", total); 437322810Shselasky else 438322810Shselasky mlx5_ib_dbg(dev, "done with all pending requests\n"); 439322810Shselasky} 440322810Shselasky 441322810Shselaskyint mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) 442322810Shselasky{ 443322810Shselasky int i; 444322810Shselasky 445322810Shselasky dev->cache.stopped = 1; 446322810Shselasky flush_workqueue(dev->cache.wq); 447322810Shselasky mlx5_mr_sysfs_cleanup(dev); 448322810Shselasky 449322810Shselasky for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) 450322810Shselasky clean_keys(dev, i); 451322810Shselasky 452322810Shselasky destroy_workqueue(dev->cache.wq); 453322810Shselasky wait_for_async_commands(dev); 454322810Shselasky del_timer_sync(&dev->delay_timer); 455322810Shselasky return 0; 456322810Shselasky} 457322810Shselasky 458322810Shselaskystruct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) 459322810Shselasky{ 460322810Shselasky struct mlx5_ib_dev *dev = to_mdev(pd->device); 461322810Shselasky struct mlx5_core_dev *mdev = dev->mdev; 462322810Shselasky struct mlx5_create_mkey_mbox_in *in; 463322810Shselasky struct mlx5_mkey_seg *seg; 464322810Shselasky struct mlx5_ib_mr *mr; 465322810Shselasky int err; 466322810Shselasky 467322810Shselasky mr = kzalloc(sizeof(*mr), GFP_KERNEL); 468322810Shselasky if (!mr) 469322810Shselasky return ERR_PTR(-ENOMEM); 470322810Shselasky 471322810Shselasky in = kzalloc(sizeof(*in), GFP_KERNEL); 472322810Shselasky if (!in) { 473322810Shselasky err = -ENOMEM; 474322810Shselasky goto err_free; 475322810Shselasky } 476322810Shselasky 477322810Shselasky seg = &in->seg; 478322810Shselasky seg->flags = convert_access(acc) | MLX5_ACCESS_MODE_PA; 479322810Shselasky seg->flags_pd = cpu_to_be32(to_mpd(pd)->pdn | MLX5_MKEY_LEN64); 480322810Shselasky seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 481322810Shselasky seg->start_addr = 0; 482322810Shselasky 483322810Shselasky err = mlx5_core_create_mkey(mdev, &mr->mmr, in, sizeof(*in), NULL, NULL, 484322810Shselasky NULL); 485322810Shselasky if (err) 486322810Shselasky goto err_in; 487322810Shselasky 488322810Shselasky kfree(in); 489322810Shselasky mr->ibmr.lkey = mr->mmr.key; 490322810Shselasky mr->ibmr.rkey = mr->mmr.key; 491322810Shselasky mr->umem = NULL; 492322810Shselasky 493322810Shselasky return &mr->ibmr; 494322810Shselasky 495322810Shselaskyerr_in: 496322810Shselasky kfree(in); 497322810Shselasky 498322810Shselaskyerr_free: 499322810Shselasky kfree(mr); 500322810Shselasky 501322810Shselasky return ERR_PTR(err); 502322810Shselasky} 503322810Shselasky 504322810Shselaskystatic int get_octo_len(u64 addr, u64 len, u64 page_size) 505322810Shselasky{ 506322810Shselasky u64 offset; 507322810Shselasky int npages; 508322810Shselasky 509322810Shselasky offset = addr & (page_size - 1ULL); 510322810Shselasky npages = ALIGN(len + offset, page_size) >> ilog2(page_size); 511322810Shselasky return (npages + 1) / 2; 512322810Shselasky} 513322810Shselasky 514322810Shselaskyvoid mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context) 515322810Shselasky{ 516322810Shselasky struct mlx5_ib_umr_context *context; 517322810Shselasky struct ib_wc wc; 518322810Shselasky int err; 519322810Shselasky 520322810Shselasky while (1) { 521322810Shselasky err = ib_poll_cq(cq, 1, &wc); 522322810Shselasky if (err < 0) { 523322810Shselasky printf("mlx5_ib: WARN: ""poll cq error %d\n", err); 524322810Shselasky return; 525322810Shselasky } 526322810Shselasky if (err == 0) 527322810Shselasky break; 528322810Shselasky 529322810Shselasky context = (struct mlx5_ib_umr_context *)(uintptr_t)wc.wr_id; 530322810Shselasky context->status = wc.status; 531322810Shselasky complete(&context->done); 532322810Shselasky } 533322810Shselasky ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 534322810Shselasky} 535322810Shselasky 536322810Shselaskystatic struct mlx5_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr, 537322810Shselasky u64 length, struct ib_umem *umem, 538322810Shselasky int npages, int page_shift, 539322810Shselasky int access_flags) 540322810Shselasky{ 541322810Shselasky struct mlx5_ib_dev *dev = to_mdev(pd->device); 542322810Shselasky struct mlx5_create_mkey_mbox_in *in; 543322810Shselasky struct mlx5_ib_mr *mr; 544322810Shselasky int inlen; 545322810Shselasky int err; 546322810Shselasky bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)); 547322810Shselasky 548322810Shselasky mr = kzalloc(sizeof(*mr), GFP_KERNEL); 549322810Shselasky if (!mr) 550322810Shselasky return ERR_PTR(-ENOMEM); 551322810Shselasky 552322810Shselasky inlen = sizeof(*in) + sizeof(*in->pas) * ((npages + 1) / 2) * 2; 553322810Shselasky in = mlx5_vzalloc(inlen); 554322810Shselasky if (!in) { 555322810Shselasky err = -ENOMEM; 556322810Shselasky goto err_1; 557322810Shselasky } 558322810Shselasky mlx5_ib_populate_pas(dev, umem, page_shift, in->pas, 559322810Shselasky pg_cap ? MLX5_IB_MTT_PRESENT : 0); 560322810Shselasky 561322810Shselasky /* The MLX5_MKEY_INBOX_PG_ACCESS bit allows setting the access flags 562322810Shselasky * in the page list submitted with the command. */ 563322810Shselasky in->flags = pg_cap ? cpu_to_be32(MLX5_MKEY_INBOX_PG_ACCESS) : 0; 564322810Shselasky in->seg.flags = convert_access(access_flags) | 565322810Shselasky MLX5_ACCESS_MODE_MTT; 566322810Shselasky in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn); 567322810Shselasky in->seg.start_addr = cpu_to_be64(virt_addr); 568322810Shselasky in->seg.len = cpu_to_be64(length); 569322810Shselasky in->seg.bsfs_octo_size = 0; 570322810Shselasky in->seg.xlt_oct_size = cpu_to_be32(get_octo_len(virt_addr, length, 1 << page_shift)); 571322810Shselasky in->seg.log2_page_size = page_shift; 572322810Shselasky in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 573322810Shselasky in->xlat_oct_act_size = cpu_to_be32(get_octo_len(virt_addr, length, 574322810Shselasky 1 << page_shift)); 575322810Shselasky err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, inlen, NULL, 576322810Shselasky NULL, NULL); 577322810Shselasky if (err) { 578322810Shselasky mlx5_ib_warn(dev, "create mkey failed\n"); 579322810Shselasky goto err_2; 580322810Shselasky } 581322810Shselasky mr->umem = umem; 582322810Shselasky mr->dev = dev; 583322810Shselasky kvfree(in); 584322810Shselasky 585322810Shselasky mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmr.key); 586322810Shselasky 587322810Shselasky return mr; 588322810Shselasky 589322810Shselaskyerr_2: 590322810Shselasky kvfree(in); 591322810Shselasky 592322810Shselaskyerr_1: 593322810Shselasky kfree(mr); 594322810Shselasky 595322810Shselasky return ERR_PTR(err); 596322810Shselasky} 597322810Shselasky 598322810Shselaskyenum { 599322810Shselasky MLX5_MAX_REG_ORDER = MAX_MR_CACHE_ENTRIES + 1, 600322810Shselasky MLX5_MAX_REG_SIZE = 2ul * 1024 * 1024 * 1024, 601322810Shselasky}; 602322810Shselasky 603322810Shselaskystatic int clean_mr(struct mlx5_ib_mr *mr) 604322810Shselasky{ 605322810Shselasky struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); 606322810Shselasky int umred = mr->umred; 607322810Shselasky int err; 608322810Shselasky int i; 609322810Shselasky 610322810Shselasky if (!umred) { 611322810Shselasky for (i = 0; i < mr->nchild; ++i) { 612322810Shselasky free_cached_mr(dev, mr->children[i]); 613322810Shselasky } 614322810Shselasky kfree(mr->children); 615322810Shselasky 616322810Shselasky err = destroy_mkey(dev, mr); 617322810Shselasky if (err) { 618322810Shselasky mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n", 619322810Shselasky mr->mmr.key, err); 620322810Shselasky return err; 621322810Shselasky } 622322810Shselasky } 623322810Shselasky return 0; 624322810Shselasky} 625322810Shselasky 626322810Shselaskystruct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 627322810Shselasky u64 virt_addr, int access_flags, 628322810Shselasky struct ib_udata *udata, int mr_id) 629322810Shselasky{ 630322810Shselasky struct mlx5_ib_dev *dev = to_mdev(pd->device); 631322810Shselasky struct mlx5_ib_mr *mr = NULL; 632322810Shselasky struct ib_umem *umem; 633322810Shselasky int page_shift; 634322810Shselasky int npages; 635322810Shselasky int ncont; 636322810Shselasky int order; 637322810Shselasky int err; 638322810Shselasky 639322810Shselasky mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 640322810Shselasky (unsigned long long)start, (unsigned long long)virt_addr, 641322810Shselasky (unsigned long long)length, access_flags); 642322810Shselasky umem = ib_umem_get(pd->uobject->context, start, length, access_flags, 0); 643322810Shselasky if (IS_ERR(umem)) { 644322810Shselasky mlx5_ib_warn(dev, "umem get failed (%ld)\n", PTR_ERR(umem)); 645322810Shselasky return (void *)umem; 646322810Shselasky } 647322810Shselasky 648322810Shselasky mlx5_ib_cont_pages(umem, start, &npages, &page_shift, &ncont, &order); 649322810Shselasky if (!npages) { 650322810Shselasky mlx5_ib_warn(dev, "avoid zero region\n"); 651322810Shselasky err = -EINVAL; 652322810Shselasky goto error; 653322810Shselasky } 654322810Shselasky 655322810Shselasky mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n", 656322810Shselasky npages, ncont, order, page_shift); 657322810Shselasky 658322810Shselasky mutex_lock(&dev->slow_path_mutex); 659322810Shselasky mr = reg_create(pd, virt_addr, length, umem, ncont, page_shift, access_flags); 660322810Shselasky mutex_unlock(&dev->slow_path_mutex); 661322810Shselasky 662322810Shselasky if (IS_ERR(mr)) { 663322810Shselasky err = PTR_ERR(mr); 664322810Shselasky mr = NULL; 665322810Shselasky goto error; 666322810Shselasky } 667322810Shselasky 668322810Shselasky mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmr.key); 669322810Shselasky 670322810Shselasky mr->umem = umem; 671322810Shselasky mr->npages = npages; 672322810Shselasky atomic_add(npages, &dev->mdev->priv.reg_pages); 673322810Shselasky mr->ibmr.lkey = mr->mmr.key; 674322810Shselasky mr->ibmr.rkey = mr->mmr.key; 675322810Shselasky 676322810Shselasky return &mr->ibmr; 677322810Shselasky 678322810Shselaskyerror: 679322810Shselasky /* 680322810Shselasky * Destroy the umem *before* destroying the MR, to ensure we 681322810Shselasky * will not have any in-flight notifiers when destroying the 682322810Shselasky * MR. 683322810Shselasky * 684322810Shselasky * As the MR is completely invalid to begin with, and this 685322810Shselasky * error path is only taken if we can't push the mr entry into 686322810Shselasky * the pagefault tree, this is safe. 687322810Shselasky */ 688322810Shselasky 689322810Shselasky ib_umem_release(umem); 690322810Shselasky return ERR_PTR(err); 691322810Shselasky} 692322810Shselasky 693322810ShselaskyCTASSERT(sizeof(((struct ib_phys_buf *)0)->size) == 8); 694322810Shselasky 695322810Shselaskystruct ib_mr * 696322810Shselaskymlx5_ib_reg_phys_mr(struct ib_pd *pd, 697322810Shselasky struct ib_phys_buf *buffer_list, 698322810Shselasky int num_phys_buf, 699322810Shselasky int access_flags, 700322810Shselasky u64 *virt_addr) 701322810Shselasky{ 702322810Shselasky struct mlx5_ib_dev *dev = to_mdev(pd->device); 703322810Shselasky struct mlx5_create_mkey_mbox_in *in; 704322810Shselasky struct mlx5_ib_mr *mr; 705322810Shselasky u64 total_size; 706322810Shselasky u32 octo_len; 707322810Shselasky bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)); 708322810Shselasky unsigned long mask; 709322810Shselasky int shift; 710322810Shselasky int npages; 711322810Shselasky int inlen; 712322810Shselasky int err; 713322810Shselasky int i, j, n; 714322810Shselasky 715322810Shselasky mask = buffer_list[0].addr ^ *virt_addr; 716322810Shselasky total_size = 0; 717322810Shselasky for (i = 0; i < num_phys_buf; ++i) { 718322810Shselasky if (i != 0) 719322810Shselasky mask |= buffer_list[i].addr; 720322810Shselasky if (i != num_phys_buf - 1) 721322810Shselasky mask |= buffer_list[i].addr + buffer_list[i].size; 722322810Shselasky 723322810Shselasky total_size += buffer_list[i].size; 724322810Shselasky } 725322810Shselasky 726322810Shselasky if (mask & ~PAGE_MASK) 727322810Shselasky return ERR_PTR(-EINVAL); 728322810Shselasky 729322810Shselasky shift = __ffs(mask | 1 << 31); 730322810Shselasky 731322810Shselasky buffer_list[0].size += buffer_list[0].addr & ((1ULL << shift) - 1); 732322810Shselasky buffer_list[0].addr &= ~0ULL << shift; 733322810Shselasky 734322810Shselasky npages = 0; 735322810Shselasky for (i = 0; i < num_phys_buf; ++i) 736322810Shselasky npages += (buffer_list[i].size + (1ULL << shift) - 1) >> shift; 737322810Shselasky 738322810Shselasky if (!npages) { 739322810Shselasky mlx5_ib_warn(dev, "avoid zero region\n"); 740322810Shselasky return ERR_PTR(-EINVAL); 741322810Shselasky } 742322810Shselasky 743322810Shselasky mr = kzalloc(sizeof *mr, GFP_KERNEL); 744322810Shselasky if (!mr) 745322810Shselasky return ERR_PTR(-ENOMEM); 746322810Shselasky 747322810Shselasky octo_len = get_octo_len(*virt_addr, total_size, 1ULL << shift); 748322810Shselasky octo_len = ALIGN(octo_len, 4); 749322810Shselasky 750322810Shselasky inlen = sizeof(*in) + (octo_len * 16); 751322810Shselasky in = mlx5_vzalloc(inlen); 752322810Shselasky if (!in) { 753322810Shselasky kfree(mr); 754322810Shselasky return ERR_PTR(-ENOMEM); 755322810Shselasky } 756322810Shselasky 757322810Shselasky n = 0; 758322810Shselasky for (i = 0; i < num_phys_buf; ++i) { 759322810Shselasky for (j = 0; 760322810Shselasky j < (buffer_list[i].size + (1ULL << shift) - 1) >> shift; 761322810Shselasky ++j) { 762322810Shselasky u64 temp = buffer_list[i].addr + ((u64) j << shift); 763322810Shselasky if (pg_cap) 764322810Shselasky temp |= MLX5_IB_MTT_PRESENT; 765322810Shselasky in->pas[n++] = cpu_to_be64(temp); 766322810Shselasky } 767322810Shselasky } 768322810Shselasky 769322810Shselasky /* The MLX5_MKEY_INBOX_PG_ACCESS bit allows setting the access flags 770322810Shselasky * in the page list submitted with the command. */ 771322810Shselasky in->flags = pg_cap ? cpu_to_be32(MLX5_MKEY_INBOX_PG_ACCESS) : 0; 772322810Shselasky in->seg.flags = convert_access(access_flags) | 773322810Shselasky MLX5_ACCESS_MODE_MTT; 774322810Shselasky in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn); 775322810Shselasky in->seg.start_addr = cpu_to_be64(*virt_addr); 776322810Shselasky in->seg.len = cpu_to_be64(total_size); 777322810Shselasky in->seg.bsfs_octo_size = 0; 778322810Shselasky in->seg.xlt_oct_size = cpu_to_be32(octo_len); 779322810Shselasky in->seg.log2_page_size = shift; 780322810Shselasky in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 781322810Shselasky in->xlat_oct_act_size = cpu_to_be32(octo_len); 782322810Shselasky err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, inlen, NULL, 783322810Shselasky NULL, NULL); 784322810Shselasky mr->umem = NULL; 785322810Shselasky mr->dev = dev; 786322810Shselasky mr->npages = npages; 787322810Shselasky mr->ibmr.lkey = mr->mmr.key; 788322810Shselasky mr->ibmr.rkey = mr->mmr.key; 789322810Shselasky 790322810Shselasky kvfree(in); 791322810Shselasky 792322810Shselasky if (err) { 793322810Shselasky kfree(mr); 794322810Shselasky return ERR_PTR(err); 795322810Shselasky } 796322810Shselasky return &mr->ibmr; 797322810Shselasky} 798322810Shselasky 799322810Shselaskyint mlx5_ib_dereg_mr(struct ib_mr *ibmr) 800322810Shselasky{ 801322810Shselasky struct mlx5_ib_dev *dev = to_mdev(ibmr->device); 802322810Shselasky struct mlx5_ib_mr *mr = to_mmr(ibmr); 803322810Shselasky struct ib_umem *umem = mr->umem; 804322810Shselasky int npages = mr->npages; 805322810Shselasky int umred = mr->umred; 806322810Shselasky int err; 807322810Shselasky 808322810Shselasky err = clean_mr(mr); 809322810Shselasky if (err) 810322810Shselasky return err; 811322810Shselasky 812322810Shselasky if (umem) { 813322810Shselasky ib_umem_release(umem); 814322810Shselasky atomic_sub(npages, &dev->mdev->priv.reg_pages); 815322810Shselasky } 816322810Shselasky 817322810Shselasky if (umred) 818322810Shselasky free_cached_mr(dev, mr); 819322810Shselasky else 820322810Shselasky kfree(mr); 821322810Shselasky 822322810Shselasky return 0; 823322810Shselasky} 824322810Shselasky 825322810Shselaskyint mlx5_ib_destroy_mr(struct ib_mr *ibmr) 826322810Shselasky{ 827322810Shselasky struct mlx5_ib_dev *dev = to_mdev(ibmr->device); 828322810Shselasky struct mlx5_ib_mr *mr = to_mmr(ibmr); 829322810Shselasky int err; 830322810Shselasky 831322810Shselasky if (mr->sig) { 832322810Shselasky if (mlx5_core_destroy_psv(dev->mdev, 833322810Shselasky mr->sig->psv_memory.psv_idx)) 834322810Shselasky mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 835322810Shselasky mr->sig->psv_memory.psv_idx); 836322810Shselasky if (mlx5_core_destroy_psv(dev->mdev, 837322810Shselasky mr->sig->psv_wire.psv_idx)) 838322810Shselasky mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 839322810Shselasky mr->sig->psv_wire.psv_idx); 840322810Shselasky kfree(mr->sig); 841322810Shselasky } 842322810Shselasky 843322810Shselasky err = destroy_mkey(dev, mr); 844322810Shselasky if (err) { 845322810Shselasky mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n", 846322810Shselasky mr->mmr.key, err); 847322810Shselasky return err; 848322810Shselasky } 849322810Shselasky 850322810Shselasky kfree(mr); 851322810Shselasky 852322810Shselasky return err; 853322810Shselasky} 854322810Shselasky 855322810Shselaskystruct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd, 856322810Shselasky int max_page_list_len) 857322810Shselasky{ 858322810Shselasky struct mlx5_ib_dev *dev = to_mdev(pd->device); 859322810Shselasky struct mlx5_create_mkey_mbox_in *in; 860322810Shselasky struct mlx5_ib_mr *mr; 861322810Shselasky int err; 862322810Shselasky 863322810Shselasky mr = kzalloc(sizeof(*mr), GFP_KERNEL); 864322810Shselasky if (!mr) 865322810Shselasky return ERR_PTR(-ENOMEM); 866322810Shselasky 867322810Shselasky in = kzalloc(sizeof(*in), GFP_KERNEL); 868322810Shselasky if (!in) { 869322810Shselasky err = -ENOMEM; 870322810Shselasky goto err_free; 871322810Shselasky } 872322810Shselasky 873322810Shselasky in->seg.status = MLX5_MKEY_STATUS_FREE; 874322810Shselasky in->seg.xlt_oct_size = cpu_to_be32((max_page_list_len + 1) / 2); 875322810Shselasky in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 876322810Shselasky in->seg.flags = MLX5_PERM_UMR_EN | MLX5_ACCESS_MODE_MTT; 877322810Shselasky in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn); 878322810Shselasky /* 879322810Shselasky * TBD not needed - issue 197292 */ 880322810Shselasky in->seg.log2_page_size = PAGE_SHIFT; 881322810Shselasky 882322810Shselasky err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, sizeof(*in), NULL, 883322810Shselasky NULL, NULL); 884322810Shselasky kfree(in); 885322810Shselasky if (err) { 886322810Shselasky mlx5_ib_warn(dev, "failed create mkey\n"); 887322810Shselasky goto err_free; 888322810Shselasky } 889322810Shselasky 890322810Shselasky mr->ibmr.lkey = mr->mmr.key; 891322810Shselasky mr->ibmr.rkey = mr->mmr.key; 892322810Shselasky mr->umem = NULL; 893322810Shselasky 894322810Shselasky return &mr->ibmr; 895322810Shselasky 896322810Shselaskyerr_free: 897322810Shselasky kfree(mr); 898322810Shselasky return ERR_PTR(err); 899322810Shselasky} 900322810Shselasky 901322810Shselaskystruct ib_fast_reg_page_list *mlx5_ib_alloc_fast_reg_page_list(struct ib_device *ibdev, 902322810Shselasky int page_list_len) 903322810Shselasky{ 904322810Shselasky struct mlx5_ib_fast_reg_page_list *mfrpl; 905322810Shselasky int size = page_list_len * sizeof(u64); 906322810Shselasky 907322810Shselasky mfrpl = kmalloc(sizeof(*mfrpl), GFP_KERNEL); 908322810Shselasky if (!mfrpl) 909322810Shselasky return ERR_PTR(-ENOMEM); 910322810Shselasky 911322810Shselasky mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL); 912322810Shselasky if (!mfrpl->ibfrpl.page_list) 913322810Shselasky goto err_free; 914322810Shselasky 915322810Shselasky mfrpl->mapped_page_list = dma_alloc_coherent(ibdev->dma_device, 916322810Shselasky size, &mfrpl->map, 917322810Shselasky GFP_KERNEL); 918322810Shselasky if (!mfrpl->mapped_page_list) 919322810Shselasky goto err_free; 920322810Shselasky 921322810Shselasky WARN_ON(mfrpl->map & 0x3f); 922322810Shselasky 923322810Shselasky return &mfrpl->ibfrpl; 924322810Shselasky 925322810Shselaskyerr_free: 926322810Shselasky kfree(mfrpl->ibfrpl.page_list); 927322810Shselasky kfree(mfrpl); 928322810Shselasky return ERR_PTR(-ENOMEM); 929322810Shselasky} 930322810Shselasky 931322810Shselaskyvoid mlx5_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list) 932322810Shselasky{ 933322810Shselasky struct mlx5_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list); 934322810Shselasky struct mlx5_ib_dev *dev = to_mdev(page_list->device); 935322810Shselasky int size = page_list->max_page_list_len * sizeof(u64); 936322810Shselasky 937322810Shselasky dma_free_coherent(&dev->mdev->pdev->dev, size, mfrpl->mapped_page_list, 938322810Shselasky mfrpl->map); 939322810Shselasky kfree(mfrpl->ibfrpl.page_list); 940322810Shselasky kfree(mfrpl); 941322810Shselasky} 942322810Shselasky 943322810Shselaskystruct order_attribute { 944322810Shselasky struct attribute attr; 945322810Shselasky ssize_t (*show)(struct cache_order *, struct order_attribute *, char *buf); 946322810Shselasky ssize_t (*store)(struct cache_order *, struct order_attribute *, 947322810Shselasky const char *buf, size_t count); 948322810Shselasky}; 949322810Shselasky 950322810Shselaskystatic ssize_t cur_show(struct cache_order *co, struct order_attribute *oa, 951322810Shselasky char *buf) 952322810Shselasky{ 953322810Shselasky struct mlx5_ib_dev *dev = co->dev; 954322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 955322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[co->index]; 956322810Shselasky int err; 957322810Shselasky 958322810Shselasky err = snprintf(buf, 20, "%d\n", ent->cur); 959322810Shselasky return err; 960322810Shselasky} 961322810Shselasky 962322810Shselaskystatic ssize_t limit_show(struct cache_order *co, struct order_attribute *oa, 963322810Shselasky char *buf) 964322810Shselasky{ 965322810Shselasky struct mlx5_ib_dev *dev = co->dev; 966322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 967322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[co->index]; 968322810Shselasky int err; 969322810Shselasky 970322810Shselasky err = snprintf(buf, 20, "%d\n", ent->limit); 971322810Shselasky return err; 972322810Shselasky} 973322810Shselasky 974322810Shselaskystatic ssize_t limit_store(struct cache_order *co, struct order_attribute *oa, 975322810Shselasky const char *buf, size_t count) 976322810Shselasky{ 977322810Shselasky struct mlx5_ib_dev *dev = co->dev; 978322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 979322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[co->index]; 980322810Shselasky u32 var; 981322810Shselasky int err; 982322810Shselasky 983322810Shselasky#define kstrtouint(a,b,c) ({*(c) = strtol(a,0,b); 0;}) 984322810Shselasky#define kstrtoint(a,b,c) ({*(c) = strtol(a,0,b); 0;}) 985322810Shselasky 986322810Shselasky if (kstrtouint(buf, 0, &var)) 987322810Shselasky return -EINVAL; 988322810Shselasky 989322810Shselasky if (var > ent->size) 990322810Shselasky return -EINVAL; 991322810Shselasky 992322810Shselasky ent->limit = var; 993322810Shselasky 994322810Shselasky if (ent->cur < ent->limit) { 995322810Shselasky err = add_keys(dev, co->index, 2 * ent->limit - ent->cur); 996322810Shselasky if (err) 997322810Shselasky return err; 998322810Shselasky } 999322810Shselasky 1000322810Shselasky return count; 1001322810Shselasky} 1002322810Shselasky 1003322810Shselaskystatic ssize_t miss_show(struct cache_order *co, struct order_attribute *oa, 1004322810Shselasky char *buf) 1005322810Shselasky{ 1006322810Shselasky struct mlx5_ib_dev *dev = co->dev; 1007322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 1008322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[co->index]; 1009322810Shselasky int err; 1010322810Shselasky 1011322810Shselasky err = snprintf(buf, 20, "%d\n", ent->miss); 1012322810Shselasky return err; 1013322810Shselasky} 1014322810Shselasky 1015322810Shselaskystatic ssize_t miss_store(struct cache_order *co, struct order_attribute *oa, 1016322810Shselasky const char *buf, size_t count) 1017322810Shselasky{ 1018322810Shselasky struct mlx5_ib_dev *dev = co->dev; 1019322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 1020322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[co->index]; 1021322810Shselasky u32 var; 1022322810Shselasky 1023322810Shselasky if (kstrtouint(buf, 0, &var)) 1024322810Shselasky return -EINVAL; 1025322810Shselasky 1026322810Shselasky if (var != 0) 1027322810Shselasky return -EINVAL; 1028322810Shselasky 1029322810Shselasky ent->miss = var; 1030322810Shselasky 1031322810Shselasky return count; 1032322810Shselasky} 1033322810Shselasky 1034322810Shselaskystatic ssize_t size_show(struct cache_order *co, struct order_attribute *oa, 1035322810Shselasky char *buf) 1036322810Shselasky{ 1037322810Shselasky struct mlx5_ib_dev *dev = co->dev; 1038322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 1039322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[co->index]; 1040322810Shselasky int err; 1041322810Shselasky 1042322810Shselasky err = snprintf(buf, 20, "%d\n", ent->size); 1043322810Shselasky return err; 1044322810Shselasky} 1045322810Shselasky 1046322810Shselaskystatic ssize_t size_store(struct cache_order *co, struct order_attribute *oa, 1047322810Shselasky const char *buf, size_t count) 1048322810Shselasky{ 1049322810Shselasky struct mlx5_ib_dev *dev = co->dev; 1050322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 1051322810Shselasky struct mlx5_cache_ent *ent = &cache->ent[co->index]; 1052322810Shselasky u32 var; 1053322810Shselasky int err; 1054322810Shselasky 1055322810Shselasky if (kstrtouint(buf, 0, &var)) 1056322810Shselasky return -EINVAL; 1057322810Shselasky 1058322810Shselasky if (var < ent->limit) 1059322810Shselasky return -EINVAL; 1060322810Shselasky 1061322810Shselasky if (var > ent->size) { 1062322810Shselasky do { 1063322810Shselasky err = add_keys(dev, co->index, var - ent->size); 1064322810Shselasky if (err && err != -EAGAIN) 1065322810Shselasky return err; 1066322810Shselasky 1067322810Shselasky usleep_range(3000, 5000); 1068322810Shselasky } while (err); 1069322810Shselasky } else if (var < ent->size) { 1070322810Shselasky remove_keys(dev, co->index, ent->size - var); 1071322810Shselasky } 1072322810Shselasky 1073322810Shselasky return count; 1074322810Shselasky} 1075322810Shselasky 1076322810Shselaskystatic ssize_t order_attr_show(struct kobject *kobj, 1077322810Shselasky struct attribute *attr, char *buf) 1078322810Shselasky{ 1079322810Shselasky struct order_attribute *oa = 1080322810Shselasky container_of(attr, struct order_attribute, attr); 1081322810Shselasky struct cache_order *co = container_of(kobj, struct cache_order, kobj); 1082322810Shselasky 1083322810Shselasky if (!oa->show) 1084322810Shselasky return -EIO; 1085322810Shselasky 1086322810Shselasky return oa->show(co, oa, buf); 1087322810Shselasky} 1088322810Shselasky 1089322810Shselaskystatic ssize_t order_attr_store(struct kobject *kobj, 1090322810Shselasky struct attribute *attr, const char *buf, size_t size) 1091322810Shselasky{ 1092322810Shselasky struct order_attribute *oa = 1093322810Shselasky container_of(attr, struct order_attribute, attr); 1094322810Shselasky struct cache_order *co = container_of(kobj, struct cache_order, kobj); 1095322810Shselasky 1096322810Shselasky if (!oa->store) 1097322810Shselasky return -EIO; 1098322810Shselasky 1099322810Shselasky return oa->store(co, oa, buf, size); 1100322810Shselasky} 1101322810Shselasky 1102322810Shselaskystatic const struct sysfs_ops order_sysfs_ops = { 1103322810Shselasky .show = order_attr_show, 1104322810Shselasky .store = order_attr_store, 1105322810Shselasky}; 1106322810Shselasky 1107322810Shselasky#define ORDER_ATTR(_name) struct order_attribute order_attr_##_name = \ 1108322810Shselasky __ATTR(_name, 0644, _name##_show, _name##_store) 1109322810Shselasky#define ORDER_ATTR_RO(_name) struct order_attribute order_attr_##_name = \ 1110322810Shselasky __ATTR(_name, 0444, _name##_show, NULL) 1111322810Shselasky 1112322810Shselaskystatic ORDER_ATTR_RO(cur); 1113322810Shselaskystatic ORDER_ATTR(limit); 1114322810Shselaskystatic ORDER_ATTR(miss); 1115322810Shselaskystatic ORDER_ATTR(size); 1116322810Shselasky 1117322810Shselaskystatic struct attribute *order_default_attrs[] = { 1118322810Shselasky &order_attr_cur.attr, 1119322810Shselasky &order_attr_limit.attr, 1120322810Shselasky &order_attr_miss.attr, 1121322810Shselasky &order_attr_size.attr, 1122322810Shselasky NULL 1123322810Shselasky}; 1124322810Shselasky 1125322810Shselaskystatic struct kobj_type order_type = { 1126322810Shselasky .sysfs_ops = &order_sysfs_ops, 1127322810Shselasky .default_attrs = order_default_attrs 1128322810Shselasky}; 1129322810Shselasky 1130322810Shselasky 1131322810Shselasky 1132322810Shselaskystruct cache_attribute { 1133322810Shselasky struct attribute attr; 1134322810Shselasky ssize_t (*show)(struct mlx5_ib_dev *dev, char *buf); 1135322810Shselasky ssize_t (*store)(struct mlx5_ib_dev *dev, const char *buf, size_t count); 1136322810Shselasky}; 1137322810Shselasky 1138322810Shselaskystatic ssize_t rel_imm_show(struct mlx5_ib_dev *dev, char *buf) 1139322810Shselasky{ 1140322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 1141322810Shselasky int err; 1142322810Shselasky 1143322810Shselasky err = snprintf(buf, 20, "%d\n", cache->rel_imm); 1144322810Shselasky return err; 1145322810Shselasky} 1146322810Shselasky 1147322810Shselaskystatic ssize_t rel_imm_store(struct mlx5_ib_dev *dev, const char *buf, size_t count) 1148322810Shselasky{ 1149322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 1150322810Shselasky u32 var; 1151322810Shselasky int i; 1152322810Shselasky int found = 0; 1153322810Shselasky 1154322810Shselasky if (kstrtouint(buf, 0, &var)) 1155322810Shselasky return -EINVAL; 1156322810Shselasky 1157322810Shselasky if (var > 1) 1158322810Shselasky return -EINVAL; 1159322810Shselasky 1160322810Shselasky if (var == cache->rel_imm) 1161322810Shselasky return count; 1162322810Shselasky 1163322810Shselasky cache->rel_imm = var; 1164322810Shselasky if (cache->rel_imm == 1) { 1165322810Shselasky for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 1166322810Shselasky if (cache->ent[i].cur > 2 * cache->ent[i].limit) { 1167322810Shselasky queue_work(cache->wq, &cache->ent[i].work); 1168322810Shselasky found = 1; 1169322810Shselasky } 1170322810Shselasky } 1171322810Shselasky if (!found) 1172322810Shselasky cache->rel_imm = 0; 1173322810Shselasky } 1174322810Shselasky 1175322810Shselasky return count; 1176322810Shselasky} 1177322810Shselaskystatic ssize_t rel_timeout_show(struct mlx5_ib_dev *dev, char *buf) 1178322810Shselasky{ 1179322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 1180322810Shselasky int err; 1181322810Shselasky 1182322810Shselasky err = snprintf(buf, 20, "%d\n", cache->rel_timeout); 1183322810Shselasky return err; 1184322810Shselasky} 1185322810Shselasky 1186322810Shselaskystatic ssize_t rel_timeout_store(struct mlx5_ib_dev *dev, const char *buf, size_t count) 1187322810Shselasky{ 1188322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 1189322810Shselasky int var; 1190322810Shselasky int i; 1191322810Shselasky 1192322810Shselasky if (kstrtoint(buf, 0, &var)) 1193322810Shselasky return -EINVAL; 1194322810Shselasky 1195322810Shselasky if (var < -1 || var > MAX_MR_RELEASE_TIMEOUT) 1196322810Shselasky return -EINVAL; 1197322810Shselasky 1198322810Shselasky if (var == cache->rel_timeout) 1199322810Shselasky return count; 1200322810Shselasky 1201322810Shselasky if (cache->rel_timeout == -1 || (var < cache->rel_timeout && var != -1)) { 1202322810Shselasky cache->rel_timeout = var; 1203322810Shselasky for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 1204322810Shselasky if (cache->ent[i].cur > 2 * cache->ent[i].limit) 1205322810Shselasky queue_work(cache->wq, &cache->ent[i].work); 1206322810Shselasky } 1207322810Shselasky } else { 1208322810Shselasky cache->rel_timeout = var; 1209322810Shselasky } 1210322810Shselasky 1211322810Shselasky return count; 1212322810Shselasky} 1213322810Shselasky 1214322810Shselaskystatic ssize_t cache_attr_show(struct kobject *kobj, 1215322810Shselasky struct attribute *attr, char *buf) 1216322810Shselasky{ 1217322810Shselasky struct cache_attribute *ca = 1218322810Shselasky container_of(attr, struct cache_attribute, attr); 1219322810Shselasky struct mlx5_ib_dev *dev = container_of(kobj, struct mlx5_ib_dev, mr_cache); 1220322810Shselasky 1221322810Shselasky if (!ca->show) 1222322810Shselasky return -EIO; 1223322810Shselasky 1224322810Shselasky return ca->show(dev, buf); 1225322810Shselasky} 1226322810Shselasky 1227322810Shselaskystatic ssize_t cache_attr_store(struct kobject *kobj, 1228322810Shselasky struct attribute *attr, const char *buf, size_t size) 1229322810Shselasky{ 1230322810Shselasky struct cache_attribute *ca = 1231322810Shselasky container_of(attr, struct cache_attribute, attr); 1232322810Shselasky struct mlx5_ib_dev *dev = container_of(kobj, struct mlx5_ib_dev, mr_cache); 1233322810Shselasky 1234322810Shselasky if (!ca->store) 1235322810Shselasky return -EIO; 1236322810Shselasky 1237322810Shselasky return ca->store(dev, buf, size); 1238322810Shselasky} 1239322810Shselasky 1240322810Shselaskystatic const struct sysfs_ops cache_sysfs_ops = { 1241322810Shselasky .show = cache_attr_show, 1242322810Shselasky .store = cache_attr_store, 1243322810Shselasky}; 1244322810Shselasky 1245322810Shselasky#define CACHE_ATTR(_name) struct cache_attribute cache_attr_##_name = \ 1246322810Shselasky __ATTR(_name, 0644, _name##_show, _name##_store) 1247322810Shselasky 1248322810Shselaskystatic CACHE_ATTR(rel_imm); 1249322810Shselaskystatic CACHE_ATTR(rel_timeout); 1250322810Shselasky 1251322810Shselaskystatic struct attribute *cache_default_attrs[] = { 1252322810Shselasky &cache_attr_rel_imm.attr, 1253322810Shselasky &cache_attr_rel_timeout.attr, 1254322810Shselasky NULL 1255322810Shselasky}; 1256322810Shselasky 1257322810Shselaskystatic struct kobj_type cache_type = { 1258322810Shselasky .sysfs_ops = &cache_sysfs_ops, 1259322810Shselasky .default_attrs = cache_default_attrs 1260322810Shselasky}; 1261322810Shselasky 1262322810Shselaskystatic int mlx5_mr_sysfs_init(struct mlx5_ib_dev *dev) 1263322810Shselasky{ 1264322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 1265322810Shselasky struct device *device = &dev->ib_dev.dev; 1266322810Shselasky struct cache_order *co; 1267322810Shselasky int o; 1268322810Shselasky int i; 1269322810Shselasky int err; 1270322810Shselasky 1271322810Shselasky err = kobject_init_and_add(&dev->mr_cache, &cache_type, 1272322810Shselasky &device->kobj, "mr_cache"); 1273322810Shselasky if (err) 1274322810Shselasky return -ENOMEM; 1275322810Shselasky 1276322810Shselasky for (o = 2, i = 0; i < MAX_MR_CACHE_ENTRIES; o++, i++) { 1277322810Shselasky co = &cache->ent[i].co; 1278322810Shselasky co->order = o; 1279322810Shselasky co->index = i; 1280322810Shselasky co->dev = dev; 1281322810Shselasky err = kobject_init_and_add(&co->kobj, &order_type, 1282322810Shselasky &dev->mr_cache, "%d", o); 1283322810Shselasky if (err) 1284322810Shselasky goto err_put; 1285322810Shselasky } 1286322810Shselasky 1287322810Shselasky return 0; 1288322810Shselasky 1289322810Shselaskyerr_put: 1290322810Shselasky for (; i >= 0; i--) { 1291322810Shselasky co = &cache->ent[i].co; 1292322810Shselasky kobject_put(&co->kobj); 1293322810Shselasky } 1294322810Shselasky kobject_put(&dev->mr_cache); 1295322810Shselasky 1296322810Shselasky return err; 1297322810Shselasky} 1298322810Shselasky 1299322810Shselaskystatic void mlx5_mr_sysfs_cleanup(struct mlx5_ib_dev *dev) 1300322810Shselasky{ 1301322810Shselasky struct mlx5_mr_cache *cache = &dev->cache; 1302322810Shselasky struct cache_order *co; 1303322810Shselasky int i; 1304322810Shselasky 1305322810Shselasky for (i = MAX_MR_CACHE_ENTRIES - 1; i >= 0; i--) { 1306322810Shselasky co = &cache->ent[i].co; 1307322810Shselasky kobject_put(&co->kobj); 1308322810Shselasky } 1309322810Shselasky kobject_put(&dev->mr_cache); 1310322810Shselasky} 1311