mlx5_pagealloc.c revision 341926
1290650Shselasky/*- 2322150Shselasky * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. 3290650Shselasky * 4290650Shselasky * Redistribution and use in source and binary forms, with or without 5290650Shselasky * modification, are permitted provided that the following conditions 6290650Shselasky * are met: 7290650Shselasky * 1. Redistributions of source code must retain the above copyright 8290650Shselasky * notice, this list of conditions and the following disclaimer. 9290650Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10290650Shselasky * notice, this list of conditions and the following disclaimer in the 11290650Shselasky * documentation and/or other materials provided with the distribution. 12290650Shselasky * 13290650Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14290650Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15290650Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290650Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17290650Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18290650Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19290650Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20290650Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21290650Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22290650Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23290650Shselasky * SUCH DAMAGE. 24290650Shselasky * 25290650Shselasky * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c 341926 2018-12-12 12:06:25Z hselasky $ 26290650Shselasky */ 27290650Shselasky 28290650Shselasky#include <linux/kernel.h> 29290650Shselasky#include <linux/module.h> 30322144Shselasky#include <linux/delay.h> 31290650Shselasky#include <dev/mlx5/driver.h> 32290650Shselasky#include "mlx5_core.h" 33290650Shselasky 34322150ShselaskyCTASSERT((uintptr_t)PAGE_MASK > (uintptr_t)PAGE_SIZE); 35322150Shselasky 36290650Shselaskystruct mlx5_pages_req { 37290650Shselasky struct mlx5_core_dev *dev; 38290650Shselasky u16 func_id; 39290650Shselasky s32 npages; 40290650Shselasky struct work_struct work; 41290650Shselasky}; 42290650Shselasky 43290650Shselasky 44290650Shselaskyenum { 45290650Shselasky MAX_RECLAIM_TIME_MSECS = 5000, 46290650Shselasky}; 47290650Shselasky 48322150Shselaskystatic void 49322150Shselaskymlx5_fwp_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 50322150Shselasky{ 51322150Shselasky struct mlx5_fw_page *fwp; 52322150Shselasky uint8_t owned; 53290650Shselasky 54322150Shselasky fwp = (struct mlx5_fw_page *)arg; 55322150Shselasky owned = MLX5_DMA_OWNED(fwp->dev); 56322150Shselasky 57322150Shselasky if (!owned) 58322150Shselasky MLX5_DMA_LOCK(fwp->dev); 59322150Shselasky 60322150Shselasky if (error == 0) { 61322150Shselasky KASSERT(nseg == 1, ("Number of segments is different from 1")); 62322150Shselasky fwp->dma_addr = segs->ds_addr; 63322150Shselasky fwp->load_done = MLX5_LOAD_ST_SUCCESS; 64322150Shselasky } else { 65322150Shselasky fwp->load_done = MLX5_LOAD_ST_FAILURE; 66322150Shselasky } 67322150Shselasky MLX5_DMA_DONE(fwp->dev); 68322150Shselasky 69322150Shselasky if (!owned) 70322150Shselasky MLX5_DMA_UNLOCK(fwp->dev); 71322150Shselasky} 72322150Shselasky 73322150Shselaskyvoid 74322150Shselaskymlx5_fwp_flush(struct mlx5_fw_page *fwp) 75290650Shselasky{ 76322150Shselasky unsigned num = fwp->numpages; 77322150Shselasky 78322150Shselasky while (num--) 79322150Shselasky bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_PREWRITE); 80322150Shselasky} 81322150Shselasky 82322150Shselaskyvoid 83322150Shselaskymlx5_fwp_invalidate(struct mlx5_fw_page *fwp) 84322150Shselasky{ 85322150Shselasky unsigned num = fwp->numpages; 86322150Shselasky 87322150Shselasky while (num--) { 88322150Shselasky bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_POSTREAD); 89322150Shselasky bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_PREREAD); 90322150Shselasky } 91322150Shselasky} 92322150Shselasky 93322150Shselaskystruct mlx5_fw_page * 94322150Shselaskymlx5_fwp_alloc(struct mlx5_core_dev *dev, gfp_t flags, unsigned num) 95322150Shselasky{ 96322150Shselasky struct mlx5_fw_page *fwp; 97322150Shselasky unsigned x; 98322150Shselasky int err; 99322150Shselasky 100322150Shselasky /* check for special case */ 101322150Shselasky if (num == 0) { 102322150Shselasky fwp = kzalloc(sizeof(*fwp), flags); 103322150Shselasky if (fwp != NULL) 104322150Shselasky fwp->dev = dev; 105322150Shselasky return (fwp); 106322150Shselasky } 107322150Shselasky 108322150Shselasky /* we need sleeping context for this function */ 109322150Shselasky if (flags & M_NOWAIT) 110322150Shselasky return (NULL); 111322150Shselasky 112322150Shselasky fwp = kzalloc(sizeof(*fwp) * num, flags); 113322150Shselasky 114322150Shselasky /* serialize loading the DMA map(s) */ 115322150Shselasky sx_xlock(&dev->cmd.dma_sx); 116322150Shselasky 117322150Shselasky for (x = 0; x != num; x++) { 118322150Shselasky /* store pointer to MLX5 core device */ 119322150Shselasky fwp[x].dev = dev; 120322150Shselasky /* store number of pages left from the array */ 121322150Shselasky fwp[x].numpages = num - x; 122322150Shselasky 123322150Shselasky /* allocate memory */ 124322150Shselasky err = bus_dmamem_alloc(dev->cmd.dma_tag, &fwp[x].virt_addr, 125322150Shselasky BUS_DMA_WAITOK | BUS_DMA_COHERENT, &fwp[x].dma_map); 126322150Shselasky if (err != 0) 127322150Shselasky goto failure; 128322150Shselasky 129322150Shselasky /* load memory into DMA */ 130322150Shselasky MLX5_DMA_LOCK(dev); 131341924Shselasky (void) bus_dmamap_load( 132322150Shselasky dev->cmd.dma_tag, fwp[x].dma_map, fwp[x].virt_addr, 133322150Shselasky MLX5_ADAPTER_PAGE_SIZE, &mlx5_fwp_load_mem_cb, 134322150Shselasky fwp + x, BUS_DMA_WAITOK | BUS_DMA_COHERENT); 135322150Shselasky 136322150Shselasky while (fwp[x].load_done == MLX5_LOAD_ST_NONE) 137322150Shselasky MLX5_DMA_WAIT(dev); 138322150Shselasky MLX5_DMA_UNLOCK(dev); 139322150Shselasky 140322150Shselasky /* check for error */ 141322150Shselasky if (fwp[x].load_done != MLX5_LOAD_ST_SUCCESS) { 142322150Shselasky bus_dmamem_free(dev->cmd.dma_tag, fwp[x].virt_addr, 143322150Shselasky fwp[x].dma_map); 144322150Shselasky goto failure; 145322150Shselasky } 146322150Shselasky } 147322150Shselasky sx_xunlock(&dev->cmd.dma_sx); 148322150Shselasky return (fwp); 149322150Shselasky 150322150Shselaskyfailure: 151322150Shselasky while (x--) { 152322150Shselasky bus_dmamap_unload(dev->cmd.dma_tag, fwp[x].dma_map); 153322150Shselasky bus_dmamem_free(dev->cmd.dma_tag, fwp[x].virt_addr, fwp[x].dma_map); 154322150Shselasky } 155322150Shselasky sx_xunlock(&dev->cmd.dma_sx); 156341926Shselasky kfree(fwp); 157322150Shselasky return (NULL); 158322150Shselasky} 159322150Shselasky 160322150Shselaskyvoid 161322150Shselaskymlx5_fwp_free(struct mlx5_fw_page *fwp) 162322150Shselasky{ 163322150Shselasky struct mlx5_core_dev *dev; 164322150Shselasky unsigned num; 165322150Shselasky 166322150Shselasky /* be NULL safe */ 167322150Shselasky if (fwp == NULL) 168322150Shselasky return; 169322150Shselasky 170322150Shselasky /* check for special case */ 171322150Shselasky if (fwp->numpages == 0) { 172322150Shselasky kfree(fwp); 173322150Shselasky return; 174322150Shselasky } 175322150Shselasky 176322150Shselasky num = fwp->numpages; 177322150Shselasky dev = fwp->dev; 178322150Shselasky 179322150Shselasky while (num--) { 180322150Shselasky bus_dmamap_unload(dev->cmd.dma_tag, fwp[num].dma_map); 181322150Shselasky bus_dmamem_free(dev->cmd.dma_tag, fwp[num].virt_addr, fwp[num].dma_map); 182322150Shselasky } 183322150Shselasky 184322150Shselasky kfree(fwp); 185322150Shselasky} 186322150Shselasky 187322150Shselaskyu64 188322150Shselaskymlx5_fwp_get_dma(struct mlx5_fw_page *fwp, size_t offset) 189322150Shselasky{ 190322150Shselasky size_t index = (offset / MLX5_ADAPTER_PAGE_SIZE); 191322150Shselasky KASSERT(index < fwp->numpages, ("Invalid offset: %lld", (long long)offset)); 192322150Shselasky 193322150Shselasky return ((fwp + index)->dma_addr + (offset % MLX5_ADAPTER_PAGE_SIZE)); 194322150Shselasky} 195322150Shselasky 196322150Shselaskyvoid * 197322150Shselaskymlx5_fwp_get_virt(struct mlx5_fw_page *fwp, size_t offset) 198322150Shselasky{ 199322150Shselasky size_t index = (offset / MLX5_ADAPTER_PAGE_SIZE); 200322150Shselasky KASSERT(index < fwp->numpages, ("Invalid offset: %lld", (long long)offset)); 201322150Shselasky 202322150Shselasky return ((char *)(fwp + index)->virt_addr + (offset % MLX5_ADAPTER_PAGE_SIZE)); 203322150Shselasky} 204322150Shselasky 205322150Shselaskystatic int 206322150Shselaskymlx5_insert_fw_page_locked(struct mlx5_core_dev *dev, struct mlx5_fw_page *nfp) 207322150Shselasky{ 208290650Shselasky struct rb_root *root = &dev->priv.page_root; 209290650Shselasky struct rb_node **new = &root->rb_node; 210290650Shselasky struct rb_node *parent = NULL; 211322146Shselasky struct mlx5_fw_page *tfp; 212290650Shselasky 213290650Shselasky while (*new) { 214290650Shselasky parent = *new; 215322146Shselasky tfp = rb_entry(parent, struct mlx5_fw_page, rb_node); 216322150Shselasky if (tfp->dma_addr < nfp->dma_addr) 217290650Shselasky new = &parent->rb_left; 218322150Shselasky else if (tfp->dma_addr > nfp->dma_addr) 219290650Shselasky new = &parent->rb_right; 220290650Shselasky else 221322150Shselasky return (-EEXIST); 222290650Shselasky } 223290650Shselasky 224290650Shselasky rb_link_node(&nfp->rb_node, parent, new); 225290650Shselasky rb_insert_color(&nfp->rb_node, root); 226322150Shselasky return (0); 227290650Shselasky} 228290650Shselasky 229322150Shselaskystatic struct mlx5_fw_page * 230322150Shselaskymlx5_remove_fw_page_locked(struct mlx5_core_dev *dev, bus_addr_t addr) 231290650Shselasky{ 232290650Shselasky struct rb_root *root = &dev->priv.page_root; 233290650Shselasky struct rb_node *tmp = root->rb_node; 234322146Shselasky struct mlx5_fw_page *result = NULL; 235322146Shselasky struct mlx5_fw_page *tfp; 236290650Shselasky 237290650Shselasky while (tmp) { 238322146Shselasky tfp = rb_entry(tmp, struct mlx5_fw_page, rb_node); 239322150Shselasky if (tfp->dma_addr < addr) { 240290650Shselasky tmp = tmp->rb_left; 241322150Shselasky } else if (tfp->dma_addr > addr) { 242290650Shselasky tmp = tmp->rb_right; 243290650Shselasky } else { 244322150Shselasky rb_erase(&tfp->rb_node, &dev->priv.page_root); 245290650Shselasky result = tfp; 246290650Shselasky break; 247290650Shselasky } 248290650Shselasky } 249322150Shselasky return (result); 250322150Shselasky} 251290650Shselasky 252322150Shselaskystatic int 253322150Shselaskyalloc_4k(struct mlx5_core_dev *dev, u64 *addr, u16 func_id) 254322150Shselasky{ 255322150Shselasky struct mlx5_fw_page *fwp; 256322150Shselasky int err; 257322150Shselasky 258322150Shselasky fwp = mlx5_fwp_alloc(dev, GFP_KERNEL, 1); 259322150Shselasky if (fwp == NULL) 260322150Shselasky return (-ENOMEM); 261322150Shselasky 262322150Shselasky fwp->func_id = func_id; 263322150Shselasky 264322150Shselasky MLX5_DMA_LOCK(dev); 265322150Shselasky err = mlx5_insert_fw_page_locked(dev, fwp); 266322150Shselasky MLX5_DMA_UNLOCK(dev); 267322150Shselasky 268322150Shselasky if (err != 0) { 269322150Shselasky mlx5_fwp_free(fwp); 270322150Shselasky } else { 271322150Shselasky /* make sure cached data is cleaned */ 272322150Shselasky mlx5_fwp_invalidate(fwp); 273322150Shselasky 274322150Shselasky /* store DMA address */ 275322150Shselasky *addr = fwp->dma_addr; 276322150Shselasky } 277322150Shselasky return (err); 278290650Shselasky} 279290650Shselasky 280322150Shselaskystatic void 281322150Shselaskyfree_4k(struct mlx5_core_dev *dev, u64 addr) 282322150Shselasky{ 283322150Shselasky struct mlx5_fw_page *fwp; 284322150Shselasky 285322150Shselasky MLX5_DMA_LOCK(dev); 286322150Shselasky fwp = mlx5_remove_fw_page_locked(dev, addr); 287322150Shselasky MLX5_DMA_UNLOCK(dev); 288322150Shselasky 289322150Shselasky if (fwp == NULL) { 290322150Shselasky mlx5_core_warn(dev, "Cannot free 4K page at 0x%llx\n", (long long)addr); 291322150Shselasky return; 292322150Shselasky } 293322150Shselasky mlx5_fwp_free(fwp); 294322150Shselasky} 295322150Shselasky 296290650Shselaskystatic int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, 297290650Shselasky s32 *npages, int boot) 298290650Shselasky{ 299331807Shselasky u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {0}; 300331807Shselasky u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {0}; 301290650Shselasky int err; 302290650Shselasky 303290650Shselasky MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); 304331807Shselasky MLX5_SET(query_pages_in, in, op_mod, boot ? 305331807Shselasky MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES : 306331807Shselasky MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES); 307290650Shselasky 308331807Shselasky err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 309290650Shselasky if (err) 310290650Shselasky return err; 311290650Shselasky 312290650Shselasky *npages = MLX5_GET(query_pages_out, out, num_pages); 313290650Shselasky *func_id = MLX5_GET(query_pages_out, out, function_id); 314290650Shselasky 315290650Shselasky return 0; 316290650Shselasky} 317290650Shselasky 318290650Shselaskystatic int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, 319290650Shselasky int notify_fail) 320290650Shselasky{ 321331807Shselasky u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; 322331807Shselasky int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); 323290650Shselasky u64 addr; 324290650Shselasky int err; 325331807Shselasky u32 *in, *nin; 326308675Shselasky int i = 0; 327290650Shselasky 328331807Shselasky inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]); 329290650Shselasky in = mlx5_vzalloc(inlen); 330290650Shselasky if (!in) { 331290650Shselasky mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); 332308675Shselasky err = -ENOMEM; 333308675Shselasky goto out_alloc; 334290650Shselasky } 335290650Shselasky 336290650Shselasky for (i = 0; i < npages; i++) { 337322150Shselasky err = alloc_4k(dev, &addr, func_id); 338322150Shselasky if (err) 339322150Shselasky goto out_alloc; 340331807Shselasky MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, addr); 341290650Shselasky } 342290650Shselasky 343331807Shselasky MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); 344331807Shselasky MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE); 345331807Shselasky MLX5_SET(manage_pages_in, in, function_id, func_id); 346331807Shselasky MLX5_SET(manage_pages_in, in, input_num_entries, npages); 347331807Shselasky 348331807Shselasky err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 349290650Shselasky if (err) { 350290650Shselasky mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", 351290650Shselasky func_id, npages, err); 352290650Shselasky goto out_alloc; 353290650Shselasky } 354290650Shselasky dev->priv.fw_pages += npages; 355322144Shselasky dev->priv.pages_per_func[func_id] += npages; 356290650Shselasky 357290650Shselasky mlx5_core_dbg(dev, "err %d\n", err); 358290650Shselasky 359290650Shselasky goto out_free; 360290650Shselasky 361290650Shselaskyout_alloc: 362290650Shselasky if (notify_fail) { 363331807Shselasky nin = mlx5_vzalloc(inlen); 364322150Shselasky if (!nin) 365322150Shselasky goto out_4k; 366322150Shselasky 367290650Shselasky memset(&out, 0, sizeof(out)); 368331807Shselasky MLX5_SET(manage_pages_in, nin, opcode, MLX5_CMD_OP_MANAGE_PAGES); 369331807Shselasky MLX5_SET(manage_pages_in, nin, op_mod, MLX5_PAGES_CANT_GIVE); 370331807Shselasky MLX5_SET(manage_pages_in, nin, function_id, func_id); 371331807Shselasky if (mlx5_cmd_exec(dev, nin, inlen, out, sizeof(out))) 372290650Shselasky mlx5_core_warn(dev, "page notify failed\n"); 373331807Shselasky kvfree(nin); 374290650Shselasky } 375322150Shselasky 376322150Shselaskyout_4k: 377290650Shselasky for (i--; i >= 0; i--) 378331807Shselasky free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i])); 379290650Shselaskyout_free: 380290650Shselasky kvfree(in); 381290650Shselasky return err; 382290650Shselasky} 383290650Shselasky 384331817Shselaskystatic int reclaim_pages_cmd(struct mlx5_core_dev *dev, 385331817Shselasky u32 *in, int in_size, u32 *out, int out_size) 386331817Shselasky{ 387331817Shselasky struct mlx5_fw_page *fwp; 388331817Shselasky struct rb_node *p; 389331817Shselasky u32 func_id; 390331817Shselasky u32 npages; 391331817Shselasky u32 i = 0; 392331817Shselasky 393331817Shselasky if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) 394331817Shselasky return mlx5_cmd_exec(dev, in, in_size, out, out_size); 395331817Shselasky 396331817Shselasky /* No hard feelings, we want our pages back! */ 397331817Shselasky npages = MLX5_GET(manage_pages_in, in, input_num_entries); 398331817Shselasky func_id = MLX5_GET(manage_pages_in, in, function_id); 399331817Shselasky 400331817Shselasky p = rb_first(&dev->priv.page_root); 401331817Shselasky while (p && i < npages) { 402331817Shselasky fwp = rb_entry(p, struct mlx5_fw_page, rb_node); 403331817Shselasky p = rb_next(p); 404331817Shselasky if (fwp->func_id != func_id) 405331817Shselasky continue; 406331817Shselasky 407331817Shselasky MLX5_ARRAY_SET64(manage_pages_out, out, pas, i, fwp->dma_addr); 408331817Shselasky i++; 409331817Shselasky } 410331817Shselasky 411331817Shselasky MLX5_SET(manage_pages_out, out, output_num_entries, i); 412331817Shselasky return 0; 413331817Shselasky} 414331817Shselasky 415290650Shselaskystatic int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, 416290650Shselasky int *nclaimed) 417290650Shselasky{ 418331807Shselasky int outlen = MLX5_ST_SZ_BYTES(manage_pages_out); 419331807Shselasky u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0}; 420290650Shselasky int num_claimed; 421331807Shselasky u32 *out; 422290650Shselasky int err; 423290650Shselasky int i; 424290650Shselasky 425290650Shselasky if (nclaimed) 426290650Shselasky *nclaimed = 0; 427290650Shselasky 428331807Shselasky outlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); 429290650Shselasky out = mlx5_vzalloc(outlen); 430290650Shselasky if (!out) 431290650Shselasky return -ENOMEM; 432290650Shselasky 433331807Shselasky MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); 434331807Shselasky MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE); 435331807Shselasky MLX5_SET(manage_pages_in, in, function_id, func_id); 436331807Shselasky MLX5_SET(manage_pages_in, in, input_num_entries, npages); 437331807Shselasky 438290650Shselasky mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); 439331817Shselasky err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen); 440290650Shselasky if (err) { 441290650Shselasky mlx5_core_err(dev, "failed reclaiming pages\n"); 442290650Shselasky goto out_free; 443290650Shselasky } 444290650Shselasky 445331807Shselasky num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries); 446290650Shselasky if (nclaimed) 447290650Shselasky *nclaimed = num_claimed; 448290650Shselasky 449290650Shselasky dev->priv.fw_pages -= num_claimed; 450322144Shselasky dev->priv.pages_per_func[func_id] -= num_claimed; 451331807Shselasky for (i = 0; i < num_claimed; i++) 452331807Shselasky free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i])); 453290650Shselasky 454290650Shselaskyout_free: 455290650Shselasky kvfree(out); 456290650Shselasky return err; 457290650Shselasky} 458290650Shselasky 459290650Shselaskystatic void pages_work_handler(struct work_struct *work) 460290650Shselasky{ 461290650Shselasky struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); 462290650Shselasky struct mlx5_core_dev *dev = req->dev; 463290650Shselasky int err = 0; 464290650Shselasky 465290650Shselasky if (req->npages < 0) 466290650Shselasky err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); 467290650Shselasky else if (req->npages > 0) 468290650Shselasky err = give_pages(dev, req->func_id, req->npages, 1); 469290650Shselasky 470290650Shselasky if (err) 471290650Shselasky mlx5_core_warn(dev, "%s fail %d\n", 472290650Shselasky req->npages < 0 ? "reclaim" : "give", err); 473290650Shselasky 474290650Shselasky kfree(req); 475290650Shselasky} 476290650Shselasky 477290650Shselaskyvoid mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, 478290650Shselasky s32 npages) 479290650Shselasky{ 480290650Shselasky struct mlx5_pages_req *req; 481290650Shselasky 482290650Shselasky req = kzalloc(sizeof(*req), GFP_ATOMIC); 483290650Shselasky if (!req) { 484290650Shselasky mlx5_core_warn(dev, "failed to allocate pages request\n"); 485290650Shselasky return; 486290650Shselasky } 487290650Shselasky 488290650Shselasky req->dev = dev; 489290650Shselasky req->func_id = func_id; 490290650Shselasky req->npages = npages; 491290650Shselasky INIT_WORK(&req->work, pages_work_handler); 492290650Shselasky if (!queue_work(dev->priv.pg_wq, &req->work)) 493290650Shselasky mlx5_core_warn(dev, "failed to queue pages handler work\n"); 494290650Shselasky} 495290650Shselasky 496290650Shselaskyint mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) 497290650Shselasky{ 498290650Shselasky u16 uninitialized_var(func_id); 499290650Shselasky s32 uninitialized_var(npages); 500290650Shselasky int err; 501290650Shselasky 502290650Shselasky err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); 503290650Shselasky if (err) 504290650Shselasky return err; 505290650Shselasky 506290650Shselasky mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", 507290650Shselasky npages, boot ? "boot" : "init", func_id); 508290650Shselasky 509290650Shselasky return give_pages(dev, func_id, npages, 0); 510290650Shselasky} 511290650Shselasky 512290650Shselaskyenum { 513290650Shselasky MLX5_BLKS_FOR_RECLAIM_PAGES = 12 514290650Shselasky}; 515290650Shselasky 516322144Shselaskys64 mlx5_wait_for_reclaim_vfs_pages(struct mlx5_core_dev *dev) 517322144Shselasky{ 518322144Shselasky int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 519322144Shselasky s64 prevpages = 0; 520322144Shselasky s64 npages = 0; 521322144Shselasky 522322144Shselasky while (!time_after(jiffies, end)) { 523322144Shselasky /* exclude own function, VFs only */ 524322144Shselasky npages = dev->priv.fw_pages - dev->priv.pages_per_func[0]; 525322144Shselasky if (!npages) 526322144Shselasky break; 527322144Shselasky 528322144Shselasky if (npages != prevpages) 529322144Shselasky end = end + msecs_to_jiffies(100); 530322144Shselasky 531322144Shselasky prevpages = npages; 532322144Shselasky msleep(1); 533322144Shselasky } 534322144Shselasky 535322144Shselasky if (npages) 536322144Shselasky mlx5_core_warn(dev, "FW did not return all VFs pages, will cause to memory leak\n"); 537322144Shselasky 538322144Shselasky return -npages; 539322144Shselasky} 540322144Shselasky 541290650Shselaskystatic int optimal_reclaimed_pages(void) 542290650Shselasky{ 543290650Shselasky struct mlx5_cmd_prot_block *block; 544290650Shselasky struct mlx5_cmd_layout *lay; 545290650Shselasky int ret; 546290650Shselasky 547290650Shselasky ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - 548331807Shselasky MLX5_ST_SZ_BYTES(manage_pages_out)) / 549331807Shselasky MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); 550290650Shselasky 551290650Shselasky return ret; 552290650Shselasky} 553290650Shselasky 554290650Shselaskyint mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) 555290650Shselasky{ 556290650Shselasky int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 557322146Shselasky struct mlx5_fw_page *fwp; 558290650Shselasky struct rb_node *p; 559290650Shselasky int nclaimed = 0; 560290650Shselasky int err; 561290650Shselasky 562290650Shselasky do { 563290650Shselasky p = rb_first(&dev->priv.page_root); 564290650Shselasky if (p) { 565322146Shselasky fwp = rb_entry(p, struct mlx5_fw_page, rb_node); 566331817Shselasky err = reclaim_pages(dev, fwp->func_id, 567331817Shselasky optimal_reclaimed_pages(), 568331817Shselasky &nclaimed); 569331817Shselasky if (err) { 570331817Shselasky mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", 571331817Shselasky err); 572331817Shselasky return err; 573290650Shselasky } 574322148Shselasky 575290650Shselasky if (nclaimed) 576290650Shselasky end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 577290650Shselasky } 578290650Shselasky if (time_after(jiffies, end)) { 579290650Shselasky mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); 580290650Shselasky break; 581290650Shselasky } 582290650Shselasky } while (p); 583290650Shselasky 584290650Shselasky return 0; 585290650Shselasky} 586290650Shselasky 587290650Shselaskyvoid mlx5_pagealloc_init(struct mlx5_core_dev *dev) 588290650Shselasky{ 589322150Shselasky 590290650Shselasky dev->priv.page_root = RB_ROOT; 591290650Shselasky} 592290650Shselasky 593290650Shselaskyvoid mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) 594290650Shselasky{ 595290650Shselasky /* nothing */ 596290650Shselasky} 597290650Shselasky 598290650Shselaskyint mlx5_pagealloc_start(struct mlx5_core_dev *dev) 599290650Shselasky{ 600290650Shselasky dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); 601290650Shselasky if (!dev->priv.pg_wq) 602290650Shselasky return -ENOMEM; 603290650Shselasky 604290650Shselasky return 0; 605290650Shselasky} 606290650Shselasky 607290650Shselaskyvoid mlx5_pagealloc_stop(struct mlx5_core_dev *dev) 608290650Shselasky{ 609290650Shselasky destroy_workqueue(dev->priv.pg_wq); 610290650Shselasky} 611