mlx5_pagealloc.c revision 341924
1290650Shselasky/*- 2322150Shselasky * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. 3290650Shselasky * 4290650Shselasky * Redistribution and use in source and binary forms, with or without 5290650Shselasky * modification, are permitted provided that the following conditions 6290650Shselasky * are met: 7290650Shselasky * 1. Redistributions of source code must retain the above copyright 8290650Shselasky * notice, this list of conditions and the following disclaimer. 9290650Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10290650Shselasky * notice, this list of conditions and the following disclaimer in the 11290650Shselasky * documentation and/or other materials provided with the distribution. 12290650Shselasky * 13290650Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14290650Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15290650Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290650Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17290650Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18290650Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19290650Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20290650Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21290650Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22290650Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23290650Shselasky * SUCH DAMAGE. 24290650Shselasky * 25290650Shselasky * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c 341924 2018-12-12 12:04:56Z hselasky $ 26290650Shselasky */ 27290650Shselasky 28290650Shselasky#include <linux/kernel.h> 29290650Shselasky#include <linux/module.h> 30322144Shselasky#include <linux/delay.h> 31290650Shselasky#include <dev/mlx5/driver.h> 32290650Shselasky#include "mlx5_core.h" 33290650Shselasky 34322150ShselaskyCTASSERT((uintptr_t)PAGE_MASK > (uintptr_t)PAGE_SIZE); 35322150Shselasky 36290650Shselaskystruct mlx5_pages_req { 37290650Shselasky struct mlx5_core_dev *dev; 38290650Shselasky u16 func_id; 39290650Shselasky s32 npages; 40290650Shselasky struct work_struct work; 41290650Shselasky}; 42290650Shselasky 43290650Shselasky 44290650Shselaskyenum { 45290650Shselasky MAX_RECLAIM_TIME_MSECS = 5000, 46290650Shselasky}; 47290650Shselasky 48322150Shselaskystatic void 49322150Shselaskymlx5_fwp_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 50322150Shselasky{ 51322150Shselasky struct mlx5_fw_page *fwp; 52322150Shselasky uint8_t owned; 53290650Shselasky 54322150Shselasky fwp = (struct mlx5_fw_page *)arg; 55322150Shselasky owned = MLX5_DMA_OWNED(fwp->dev); 56322150Shselasky 57322150Shselasky if (!owned) 58322150Shselasky MLX5_DMA_LOCK(fwp->dev); 59322150Shselasky 60322150Shselasky if (error == 0) { 61322150Shselasky KASSERT(nseg == 1, ("Number of segments is different from 1")); 62322150Shselasky fwp->dma_addr = segs->ds_addr; 63322150Shselasky fwp->load_done = MLX5_LOAD_ST_SUCCESS; 64322150Shselasky } else { 65322150Shselasky fwp->load_done = MLX5_LOAD_ST_FAILURE; 66322150Shselasky } 67322150Shselasky MLX5_DMA_DONE(fwp->dev); 68322150Shselasky 69322150Shselasky if (!owned) 70322150Shselasky MLX5_DMA_UNLOCK(fwp->dev); 71322150Shselasky} 72322150Shselasky 73322150Shselaskyvoid 74322150Shselaskymlx5_fwp_flush(struct mlx5_fw_page *fwp) 75290650Shselasky{ 76322150Shselasky unsigned num = fwp->numpages; 77322150Shselasky 78322150Shselasky while (num--) 79322150Shselasky bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_PREWRITE); 80322150Shselasky} 81322150Shselasky 82322150Shselaskyvoid 83322150Shselaskymlx5_fwp_invalidate(struct mlx5_fw_page *fwp) 84322150Shselasky{ 85322150Shselasky unsigned num = fwp->numpages; 86322150Shselasky 87322150Shselasky while (num--) { 88322150Shselasky bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_POSTREAD); 89322150Shselasky bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_PREREAD); 90322150Shselasky } 91322150Shselasky} 92322150Shselasky 93322150Shselaskystruct mlx5_fw_page * 94322150Shselaskymlx5_fwp_alloc(struct mlx5_core_dev *dev, gfp_t flags, unsigned num) 95322150Shselasky{ 96322150Shselasky struct mlx5_fw_page *fwp; 97322150Shselasky unsigned x; 98322150Shselasky int err; 99322150Shselasky 100322150Shselasky /* check for special case */ 101322150Shselasky if (num == 0) { 102322150Shselasky fwp = kzalloc(sizeof(*fwp), flags); 103322150Shselasky if (fwp != NULL) 104322150Shselasky fwp->dev = dev; 105322150Shselasky return (fwp); 106322150Shselasky } 107322150Shselasky 108322150Shselasky /* we need sleeping context for this function */ 109322150Shselasky if (flags & M_NOWAIT) 110322150Shselasky return (NULL); 111322150Shselasky 112322150Shselasky fwp = kzalloc(sizeof(*fwp) * num, flags); 113322150Shselasky 114322150Shselasky /* serialize loading the DMA map(s) */ 115322150Shselasky sx_xlock(&dev->cmd.dma_sx); 116322150Shselasky 117322150Shselasky for (x = 0; x != num; x++) { 118322150Shselasky /* store pointer to MLX5 core device */ 119322150Shselasky fwp[x].dev = dev; 120322150Shselasky /* store number of pages left from the array */ 121322150Shselasky fwp[x].numpages = num - x; 122322150Shselasky 123322150Shselasky /* allocate memory */ 124322150Shselasky err = bus_dmamem_alloc(dev->cmd.dma_tag, &fwp[x].virt_addr, 125322150Shselasky BUS_DMA_WAITOK | BUS_DMA_COHERENT, &fwp[x].dma_map); 126322150Shselasky if (err != 0) 127322150Shselasky goto failure; 128322150Shselasky 129322150Shselasky /* load memory into DMA */ 130322150Shselasky MLX5_DMA_LOCK(dev); 131341924Shselasky (void) bus_dmamap_load( 132322150Shselasky dev->cmd.dma_tag, fwp[x].dma_map, fwp[x].virt_addr, 133322150Shselasky MLX5_ADAPTER_PAGE_SIZE, &mlx5_fwp_load_mem_cb, 134322150Shselasky fwp + x, BUS_DMA_WAITOK | BUS_DMA_COHERENT); 135322150Shselasky 136322150Shselasky while (fwp[x].load_done == MLX5_LOAD_ST_NONE) 137322150Shselasky MLX5_DMA_WAIT(dev); 138322150Shselasky MLX5_DMA_UNLOCK(dev); 139322150Shselasky 140322150Shselasky /* check for error */ 141322150Shselasky if (fwp[x].load_done != MLX5_LOAD_ST_SUCCESS) { 142322150Shselasky bus_dmamem_free(dev->cmd.dma_tag, fwp[x].virt_addr, 143322150Shselasky fwp[x].dma_map); 144322150Shselasky goto failure; 145322150Shselasky } 146322150Shselasky } 147322150Shselasky sx_xunlock(&dev->cmd.dma_sx); 148322150Shselasky return (fwp); 149322150Shselasky 150322150Shselaskyfailure: 151322150Shselasky while (x--) { 152322150Shselasky bus_dmamap_unload(dev->cmd.dma_tag, fwp[x].dma_map); 153322150Shselasky bus_dmamem_free(dev->cmd.dma_tag, fwp[x].virt_addr, fwp[x].dma_map); 154322150Shselasky } 155322150Shselasky sx_xunlock(&dev->cmd.dma_sx); 156322150Shselasky return (NULL); 157322150Shselasky} 158322150Shselasky 159322150Shselaskyvoid 160322150Shselaskymlx5_fwp_free(struct mlx5_fw_page *fwp) 161322150Shselasky{ 162322150Shselasky struct mlx5_core_dev *dev; 163322150Shselasky unsigned num; 164322150Shselasky 165322150Shselasky /* be NULL safe */ 166322150Shselasky if (fwp == NULL) 167322150Shselasky return; 168322150Shselasky 169322150Shselasky /* check for special case */ 170322150Shselasky if (fwp->numpages == 0) { 171322150Shselasky kfree(fwp); 172322150Shselasky return; 173322150Shselasky } 174322150Shselasky 175322150Shselasky num = fwp->numpages; 176322150Shselasky dev = fwp->dev; 177322150Shselasky 178322150Shselasky while (num--) { 179322150Shselasky bus_dmamap_unload(dev->cmd.dma_tag, fwp[num].dma_map); 180322150Shselasky bus_dmamem_free(dev->cmd.dma_tag, fwp[num].virt_addr, fwp[num].dma_map); 181322150Shselasky } 182322150Shselasky 183322150Shselasky kfree(fwp); 184322150Shselasky} 185322150Shselasky 186322150Shselaskyu64 187322150Shselaskymlx5_fwp_get_dma(struct mlx5_fw_page *fwp, size_t offset) 188322150Shselasky{ 189322150Shselasky size_t index = (offset / MLX5_ADAPTER_PAGE_SIZE); 190322150Shselasky KASSERT(index < fwp->numpages, ("Invalid offset: %lld", (long long)offset)); 191322150Shselasky 192322150Shselasky return ((fwp + index)->dma_addr + (offset % MLX5_ADAPTER_PAGE_SIZE)); 193322150Shselasky} 194322150Shselasky 195322150Shselaskyvoid * 196322150Shselaskymlx5_fwp_get_virt(struct mlx5_fw_page *fwp, size_t offset) 197322150Shselasky{ 198322150Shselasky size_t index = (offset / MLX5_ADAPTER_PAGE_SIZE); 199322150Shselasky KASSERT(index < fwp->numpages, ("Invalid offset: %lld", (long long)offset)); 200322150Shselasky 201322150Shselasky return ((char *)(fwp + index)->virt_addr + (offset % MLX5_ADAPTER_PAGE_SIZE)); 202322150Shselasky} 203322150Shselasky 204322150Shselaskystatic int 205322150Shselaskymlx5_insert_fw_page_locked(struct mlx5_core_dev *dev, struct mlx5_fw_page *nfp) 206322150Shselasky{ 207290650Shselasky struct rb_root *root = &dev->priv.page_root; 208290650Shselasky struct rb_node **new = &root->rb_node; 209290650Shselasky struct rb_node *parent = NULL; 210322146Shselasky struct mlx5_fw_page *tfp; 211290650Shselasky 212290650Shselasky while (*new) { 213290650Shselasky parent = *new; 214322146Shselasky tfp = rb_entry(parent, struct mlx5_fw_page, rb_node); 215322150Shselasky if (tfp->dma_addr < nfp->dma_addr) 216290650Shselasky new = &parent->rb_left; 217322150Shselasky else if (tfp->dma_addr > nfp->dma_addr) 218290650Shselasky new = &parent->rb_right; 219290650Shselasky else 220322150Shselasky return (-EEXIST); 221290650Shselasky } 222290650Shselasky 223290650Shselasky rb_link_node(&nfp->rb_node, parent, new); 224290650Shselasky rb_insert_color(&nfp->rb_node, root); 225322150Shselasky return (0); 226290650Shselasky} 227290650Shselasky 228322150Shselaskystatic struct mlx5_fw_page * 229322150Shselaskymlx5_remove_fw_page_locked(struct mlx5_core_dev *dev, bus_addr_t addr) 230290650Shselasky{ 231290650Shselasky struct rb_root *root = &dev->priv.page_root; 232290650Shselasky struct rb_node *tmp = root->rb_node; 233322146Shselasky struct mlx5_fw_page *result = NULL; 234322146Shselasky struct mlx5_fw_page *tfp; 235290650Shselasky 236290650Shselasky while (tmp) { 237322146Shselasky tfp = rb_entry(tmp, struct mlx5_fw_page, rb_node); 238322150Shselasky if (tfp->dma_addr < addr) { 239290650Shselasky tmp = tmp->rb_left; 240322150Shselasky } else if (tfp->dma_addr > addr) { 241290650Shselasky tmp = tmp->rb_right; 242290650Shselasky } else { 243322150Shselasky rb_erase(&tfp->rb_node, &dev->priv.page_root); 244290650Shselasky result = tfp; 245290650Shselasky break; 246290650Shselasky } 247290650Shselasky } 248322150Shselasky return (result); 249322150Shselasky} 250290650Shselasky 251322150Shselaskystatic int 252322150Shselaskyalloc_4k(struct mlx5_core_dev *dev, u64 *addr, u16 func_id) 253322150Shselasky{ 254322150Shselasky struct mlx5_fw_page *fwp; 255322150Shselasky int err; 256322150Shselasky 257322150Shselasky fwp = mlx5_fwp_alloc(dev, GFP_KERNEL, 1); 258322150Shselasky if (fwp == NULL) 259322150Shselasky return (-ENOMEM); 260322150Shselasky 261322150Shselasky fwp->func_id = func_id; 262322150Shselasky 263322150Shselasky MLX5_DMA_LOCK(dev); 264322150Shselasky err = mlx5_insert_fw_page_locked(dev, fwp); 265322150Shselasky MLX5_DMA_UNLOCK(dev); 266322150Shselasky 267322150Shselasky if (err != 0) { 268322150Shselasky mlx5_fwp_free(fwp); 269322150Shselasky } else { 270322150Shselasky /* make sure cached data is cleaned */ 271322150Shselasky mlx5_fwp_invalidate(fwp); 272322150Shselasky 273322150Shselasky /* store DMA address */ 274322150Shselasky *addr = fwp->dma_addr; 275322150Shselasky } 276322150Shselasky return (err); 277290650Shselasky} 278290650Shselasky 279322150Shselaskystatic void 280322150Shselaskyfree_4k(struct mlx5_core_dev *dev, u64 addr) 281322150Shselasky{ 282322150Shselasky struct mlx5_fw_page *fwp; 283322150Shselasky 284322150Shselasky MLX5_DMA_LOCK(dev); 285322150Shselasky fwp = mlx5_remove_fw_page_locked(dev, addr); 286322150Shselasky MLX5_DMA_UNLOCK(dev); 287322150Shselasky 288322150Shselasky if (fwp == NULL) { 289322150Shselasky mlx5_core_warn(dev, "Cannot free 4K page at 0x%llx\n", (long long)addr); 290322150Shselasky return; 291322150Shselasky } 292322150Shselasky mlx5_fwp_free(fwp); 293322150Shselasky} 294322150Shselasky 295290650Shselaskystatic int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, 296290650Shselasky s32 *npages, int boot) 297290650Shselasky{ 298331807Shselasky u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {0}; 299331807Shselasky u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {0}; 300290650Shselasky int err; 301290650Shselasky 302290650Shselasky MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); 303331807Shselasky MLX5_SET(query_pages_in, in, op_mod, boot ? 304331807Shselasky MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES : 305331807Shselasky MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES); 306290650Shselasky 307331807Shselasky err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 308290650Shselasky if (err) 309290650Shselasky return err; 310290650Shselasky 311290650Shselasky *npages = MLX5_GET(query_pages_out, out, num_pages); 312290650Shselasky *func_id = MLX5_GET(query_pages_out, out, function_id); 313290650Shselasky 314290650Shselasky return 0; 315290650Shselasky} 316290650Shselasky 317290650Shselaskystatic int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, 318290650Shselasky int notify_fail) 319290650Shselasky{ 320331807Shselasky u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; 321331807Shselasky int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); 322290650Shselasky u64 addr; 323290650Shselasky int err; 324331807Shselasky u32 *in, *nin; 325308675Shselasky int i = 0; 326290650Shselasky 327331807Shselasky inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]); 328290650Shselasky in = mlx5_vzalloc(inlen); 329290650Shselasky if (!in) { 330290650Shselasky mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); 331308675Shselasky err = -ENOMEM; 332308675Shselasky goto out_alloc; 333290650Shselasky } 334290650Shselasky 335290650Shselasky for (i = 0; i < npages; i++) { 336322150Shselasky err = alloc_4k(dev, &addr, func_id); 337322150Shselasky if (err) 338322150Shselasky goto out_alloc; 339331807Shselasky MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, addr); 340290650Shselasky } 341290650Shselasky 342331807Shselasky MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); 343331807Shselasky MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE); 344331807Shselasky MLX5_SET(manage_pages_in, in, function_id, func_id); 345331807Shselasky MLX5_SET(manage_pages_in, in, input_num_entries, npages); 346331807Shselasky 347331807Shselasky err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 348290650Shselasky if (err) { 349290650Shselasky mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", 350290650Shselasky func_id, npages, err); 351290650Shselasky goto out_alloc; 352290650Shselasky } 353290650Shselasky dev->priv.fw_pages += npages; 354322144Shselasky dev->priv.pages_per_func[func_id] += npages; 355290650Shselasky 356290650Shselasky mlx5_core_dbg(dev, "err %d\n", err); 357290650Shselasky 358290650Shselasky goto out_free; 359290650Shselasky 360290650Shselaskyout_alloc: 361290650Shselasky if (notify_fail) { 362331807Shselasky nin = mlx5_vzalloc(inlen); 363322150Shselasky if (!nin) 364322150Shselasky goto out_4k; 365322150Shselasky 366290650Shselasky memset(&out, 0, sizeof(out)); 367331807Shselasky MLX5_SET(manage_pages_in, nin, opcode, MLX5_CMD_OP_MANAGE_PAGES); 368331807Shselasky MLX5_SET(manage_pages_in, nin, op_mod, MLX5_PAGES_CANT_GIVE); 369331807Shselasky MLX5_SET(manage_pages_in, nin, function_id, func_id); 370331807Shselasky if (mlx5_cmd_exec(dev, nin, inlen, out, sizeof(out))) 371290650Shselasky mlx5_core_warn(dev, "page notify failed\n"); 372331807Shselasky kvfree(nin); 373290650Shselasky } 374322150Shselasky 375322150Shselaskyout_4k: 376290650Shselasky for (i--; i >= 0; i--) 377331807Shselasky free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i])); 378290650Shselaskyout_free: 379290650Shselasky kvfree(in); 380290650Shselasky return err; 381290650Shselasky} 382290650Shselasky 383331817Shselaskystatic int reclaim_pages_cmd(struct mlx5_core_dev *dev, 384331817Shselasky u32 *in, int in_size, u32 *out, int out_size) 385331817Shselasky{ 386331817Shselasky struct mlx5_fw_page *fwp; 387331817Shselasky struct rb_node *p; 388331817Shselasky u32 func_id; 389331817Shselasky u32 npages; 390331817Shselasky u32 i = 0; 391331817Shselasky 392331817Shselasky if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) 393331817Shselasky return mlx5_cmd_exec(dev, in, in_size, out, out_size); 394331817Shselasky 395331817Shselasky /* No hard feelings, we want our pages back! */ 396331817Shselasky npages = MLX5_GET(manage_pages_in, in, input_num_entries); 397331817Shselasky func_id = MLX5_GET(manage_pages_in, in, function_id); 398331817Shselasky 399331817Shselasky p = rb_first(&dev->priv.page_root); 400331817Shselasky while (p && i < npages) { 401331817Shselasky fwp = rb_entry(p, struct mlx5_fw_page, rb_node); 402331817Shselasky p = rb_next(p); 403331817Shselasky if (fwp->func_id != func_id) 404331817Shselasky continue; 405331817Shselasky 406331817Shselasky MLX5_ARRAY_SET64(manage_pages_out, out, pas, i, fwp->dma_addr); 407331817Shselasky i++; 408331817Shselasky } 409331817Shselasky 410331817Shselasky MLX5_SET(manage_pages_out, out, output_num_entries, i); 411331817Shselasky return 0; 412331817Shselasky} 413331817Shselasky 414290650Shselaskystatic int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, 415290650Shselasky int *nclaimed) 416290650Shselasky{ 417331807Shselasky int outlen = MLX5_ST_SZ_BYTES(manage_pages_out); 418331807Shselasky u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0}; 419290650Shselasky int num_claimed; 420331807Shselasky u32 *out; 421290650Shselasky int err; 422290650Shselasky int i; 423290650Shselasky 424290650Shselasky if (nclaimed) 425290650Shselasky *nclaimed = 0; 426290650Shselasky 427331807Shselasky outlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); 428290650Shselasky out = mlx5_vzalloc(outlen); 429290650Shselasky if (!out) 430290650Shselasky return -ENOMEM; 431290650Shselasky 432331807Shselasky MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); 433331807Shselasky MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE); 434331807Shselasky MLX5_SET(manage_pages_in, in, function_id, func_id); 435331807Shselasky MLX5_SET(manage_pages_in, in, input_num_entries, npages); 436331807Shselasky 437290650Shselasky mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); 438331817Shselasky err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen); 439290650Shselasky if (err) { 440290650Shselasky mlx5_core_err(dev, "failed reclaiming pages\n"); 441290650Shselasky goto out_free; 442290650Shselasky } 443290650Shselasky 444331807Shselasky num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries); 445290650Shselasky if (nclaimed) 446290650Shselasky *nclaimed = num_claimed; 447290650Shselasky 448290650Shselasky dev->priv.fw_pages -= num_claimed; 449322144Shselasky dev->priv.pages_per_func[func_id] -= num_claimed; 450331807Shselasky for (i = 0; i < num_claimed; i++) 451331807Shselasky free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i])); 452290650Shselasky 453290650Shselaskyout_free: 454290650Shselasky kvfree(out); 455290650Shselasky return err; 456290650Shselasky} 457290650Shselasky 458290650Shselaskystatic void pages_work_handler(struct work_struct *work) 459290650Shselasky{ 460290650Shselasky struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); 461290650Shselasky struct mlx5_core_dev *dev = req->dev; 462290650Shselasky int err = 0; 463290650Shselasky 464290650Shselasky if (req->npages < 0) 465290650Shselasky err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); 466290650Shselasky else if (req->npages > 0) 467290650Shselasky err = give_pages(dev, req->func_id, req->npages, 1); 468290650Shselasky 469290650Shselasky if (err) 470290650Shselasky mlx5_core_warn(dev, "%s fail %d\n", 471290650Shselasky req->npages < 0 ? "reclaim" : "give", err); 472290650Shselasky 473290650Shselasky kfree(req); 474290650Shselasky} 475290650Shselasky 476290650Shselaskyvoid mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, 477290650Shselasky s32 npages) 478290650Shselasky{ 479290650Shselasky struct mlx5_pages_req *req; 480290650Shselasky 481290650Shselasky req = kzalloc(sizeof(*req), GFP_ATOMIC); 482290650Shselasky if (!req) { 483290650Shselasky mlx5_core_warn(dev, "failed to allocate pages request\n"); 484290650Shselasky return; 485290650Shselasky } 486290650Shselasky 487290650Shselasky req->dev = dev; 488290650Shselasky req->func_id = func_id; 489290650Shselasky req->npages = npages; 490290650Shselasky INIT_WORK(&req->work, pages_work_handler); 491290650Shselasky if (!queue_work(dev->priv.pg_wq, &req->work)) 492290650Shselasky mlx5_core_warn(dev, "failed to queue pages handler work\n"); 493290650Shselasky} 494290650Shselasky 495290650Shselaskyint mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) 496290650Shselasky{ 497290650Shselasky u16 uninitialized_var(func_id); 498290650Shselasky s32 uninitialized_var(npages); 499290650Shselasky int err; 500290650Shselasky 501290650Shselasky err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); 502290650Shselasky if (err) 503290650Shselasky return err; 504290650Shselasky 505290650Shselasky mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", 506290650Shselasky npages, boot ? "boot" : "init", func_id); 507290650Shselasky 508290650Shselasky return give_pages(dev, func_id, npages, 0); 509290650Shselasky} 510290650Shselasky 511290650Shselaskyenum { 512290650Shselasky MLX5_BLKS_FOR_RECLAIM_PAGES = 12 513290650Shselasky}; 514290650Shselasky 515322144Shselaskys64 mlx5_wait_for_reclaim_vfs_pages(struct mlx5_core_dev *dev) 516322144Shselasky{ 517322144Shselasky int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 518322144Shselasky s64 prevpages = 0; 519322144Shselasky s64 npages = 0; 520322144Shselasky 521322144Shselasky while (!time_after(jiffies, end)) { 522322144Shselasky /* exclude own function, VFs only */ 523322144Shselasky npages = dev->priv.fw_pages - dev->priv.pages_per_func[0]; 524322144Shselasky if (!npages) 525322144Shselasky break; 526322144Shselasky 527322144Shselasky if (npages != prevpages) 528322144Shselasky end = end + msecs_to_jiffies(100); 529322144Shselasky 530322144Shselasky prevpages = npages; 531322144Shselasky msleep(1); 532322144Shselasky } 533322144Shselasky 534322144Shselasky if (npages) 535322144Shselasky mlx5_core_warn(dev, "FW did not return all VFs pages, will cause to memory leak\n"); 536322144Shselasky 537322144Shselasky return -npages; 538322144Shselasky} 539322144Shselasky 540290650Shselaskystatic int optimal_reclaimed_pages(void) 541290650Shselasky{ 542290650Shselasky struct mlx5_cmd_prot_block *block; 543290650Shselasky struct mlx5_cmd_layout *lay; 544290650Shselasky int ret; 545290650Shselasky 546290650Shselasky ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - 547331807Shselasky MLX5_ST_SZ_BYTES(manage_pages_out)) / 548331807Shselasky MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); 549290650Shselasky 550290650Shselasky return ret; 551290650Shselasky} 552290650Shselasky 553290650Shselaskyint mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) 554290650Shselasky{ 555290650Shselasky int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 556322146Shselasky struct mlx5_fw_page *fwp; 557290650Shselasky struct rb_node *p; 558290650Shselasky int nclaimed = 0; 559290650Shselasky int err; 560290650Shselasky 561290650Shselasky do { 562290650Shselasky p = rb_first(&dev->priv.page_root); 563290650Shselasky if (p) { 564322146Shselasky fwp = rb_entry(p, struct mlx5_fw_page, rb_node); 565331817Shselasky err = reclaim_pages(dev, fwp->func_id, 566331817Shselasky optimal_reclaimed_pages(), 567331817Shselasky &nclaimed); 568331817Shselasky if (err) { 569331817Shselasky mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", 570331817Shselasky err); 571331817Shselasky return err; 572290650Shselasky } 573322148Shselasky 574290650Shselasky if (nclaimed) 575290650Shselasky end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 576290650Shselasky } 577290650Shselasky if (time_after(jiffies, end)) { 578290650Shselasky mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); 579290650Shselasky break; 580290650Shselasky } 581290650Shselasky } while (p); 582290650Shselasky 583290650Shselasky return 0; 584290650Shselasky} 585290650Shselasky 586290650Shselaskyvoid mlx5_pagealloc_init(struct mlx5_core_dev *dev) 587290650Shselasky{ 588322150Shselasky 589290650Shselasky dev->priv.page_root = RB_ROOT; 590290650Shselasky} 591290650Shselasky 592290650Shselaskyvoid mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) 593290650Shselasky{ 594290650Shselasky /* nothing */ 595290650Shselasky} 596290650Shselasky 597290650Shselaskyint mlx5_pagealloc_start(struct mlx5_core_dev *dev) 598290650Shselasky{ 599290650Shselasky dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); 600290650Shselasky if (!dev->priv.pg_wq) 601290650Shselasky return -ENOMEM; 602290650Shselasky 603290650Shselasky return 0; 604290650Shselasky} 605290650Shselasky 606290650Shselaskyvoid mlx5_pagealloc_stop(struct mlx5_core_dev *dev) 607290650Shselasky{ 608290650Shselasky destroy_workqueue(dev->priv.pg_wq); 609290650Shselasky} 610