mlx5_pagealloc.c revision 322150
1290650Shselasky/*- 2322150Shselasky * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. 3290650Shselasky * 4290650Shselasky * Redistribution and use in source and binary forms, with or without 5290650Shselasky * modification, are permitted provided that the following conditions 6290650Shselasky * are met: 7290650Shselasky * 1. Redistributions of source code must retain the above copyright 8290650Shselasky * notice, this list of conditions and the following disclaimer. 9290650Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10290650Shselasky * notice, this list of conditions and the following disclaimer in the 11290650Shselasky * documentation and/or other materials provided with the distribution. 12290650Shselasky * 13290650Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14290650Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15290650Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290650Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17290650Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18290650Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19290650Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20290650Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21290650Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22290650Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23290650Shselasky * SUCH DAMAGE. 24290650Shselasky * 25290650Shselasky * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c 322150 2017-08-07 12:48:13Z hselasky $ 26290650Shselasky */ 27290650Shselasky 28290650Shselasky#include <linux/kernel.h> 29290650Shselasky#include <linux/module.h> 30322144Shselasky#include <linux/delay.h> 31290650Shselasky#include <dev/mlx5/driver.h> 32290650Shselasky#include "mlx5_core.h" 33290650Shselasky 34322150ShselaskyCTASSERT((uintptr_t)PAGE_MASK > (uintptr_t)PAGE_SIZE); 35322150Shselasky 36290650Shselaskystruct mlx5_pages_req { 37290650Shselasky struct mlx5_core_dev *dev; 38290650Shselasky u16 func_id; 39290650Shselasky s32 npages; 40290650Shselasky struct work_struct work; 41290650Shselasky}; 42290650Shselasky 43290650Shselasky 44290650Shselaskystruct mlx5_manage_pages_inbox { 45290650Shselasky struct mlx5_inbox_hdr hdr; 46290650Shselasky __be16 rsvd; 47290650Shselasky __be16 func_id; 48290650Shselasky __be32 num_entries; 49290650Shselasky __be64 pas[0]; 50290650Shselasky}; 51290650Shselasky 52290650Shselaskystruct mlx5_manage_pages_outbox { 53290650Shselasky struct mlx5_outbox_hdr hdr; 54290650Shselasky __be32 num_entries; 55290650Shselasky u8 rsvd[4]; 56290650Shselasky __be64 pas[0]; 57290650Shselasky}; 58290650Shselasky 59290650Shselaskyenum { 60290650Shselasky MAX_RECLAIM_TIME_MSECS = 5000, 61290650Shselasky}; 62290650Shselasky 63322150Shselaskystatic void 64322150Shselaskymlx5_fwp_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 65322150Shselasky{ 66322150Shselasky struct mlx5_fw_page *fwp; 67322150Shselasky uint8_t owned; 68290650Shselasky 69322150Shselasky fwp = (struct mlx5_fw_page *)arg; 70322150Shselasky owned = MLX5_DMA_OWNED(fwp->dev); 71322150Shselasky 72322150Shselasky if (!owned) 73322150Shselasky MLX5_DMA_LOCK(fwp->dev); 74322150Shselasky 75322150Shselasky if (error == 0) { 76322150Shselasky KASSERT(nseg == 1, ("Number of segments is different from 1")); 77322150Shselasky fwp->dma_addr = segs->ds_addr; 78322150Shselasky fwp->load_done = MLX5_LOAD_ST_SUCCESS; 79322150Shselasky } else { 80322150Shselasky fwp->load_done = MLX5_LOAD_ST_FAILURE; 81322150Shselasky } 82322150Shselasky MLX5_DMA_DONE(fwp->dev); 83322150Shselasky 84322150Shselasky if (!owned) 85322150Shselasky MLX5_DMA_UNLOCK(fwp->dev); 86322150Shselasky} 87322150Shselasky 88322150Shselaskyvoid 89322150Shselaskymlx5_fwp_flush(struct mlx5_fw_page *fwp) 90290650Shselasky{ 91322150Shselasky unsigned num = fwp->numpages; 92322150Shselasky 93322150Shselasky while (num--) 94322150Shselasky bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_PREWRITE); 95322150Shselasky} 96322150Shselasky 97322150Shselaskyvoid 98322150Shselaskymlx5_fwp_invalidate(struct mlx5_fw_page *fwp) 99322150Shselasky{ 100322150Shselasky unsigned num = fwp->numpages; 101322150Shselasky 102322150Shselasky while (num--) { 103322150Shselasky bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_POSTREAD); 104322150Shselasky bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_PREREAD); 105322150Shselasky } 106322150Shselasky} 107322150Shselasky 108322150Shselaskystruct mlx5_fw_page * 109322150Shselaskymlx5_fwp_alloc(struct mlx5_core_dev *dev, gfp_t flags, unsigned num) 110322150Shselasky{ 111322150Shselasky struct mlx5_fw_page *fwp; 112322150Shselasky unsigned x; 113322150Shselasky int err; 114322150Shselasky 115322150Shselasky /* check for special case */ 116322150Shselasky if (num == 0) { 117322150Shselasky fwp = kzalloc(sizeof(*fwp), flags); 118322150Shselasky if (fwp != NULL) 119322150Shselasky fwp->dev = dev; 120322150Shselasky return (fwp); 121322150Shselasky } 122322150Shselasky 123322150Shselasky /* we need sleeping context for this function */ 124322150Shselasky if (flags & M_NOWAIT) 125322150Shselasky return (NULL); 126322150Shselasky 127322150Shselasky fwp = kzalloc(sizeof(*fwp) * num, flags); 128322150Shselasky 129322150Shselasky /* serialize loading the DMA map(s) */ 130322150Shselasky sx_xlock(&dev->cmd.dma_sx); 131322150Shselasky 132322150Shselasky for (x = 0; x != num; x++) { 133322150Shselasky /* store pointer to MLX5 core device */ 134322150Shselasky fwp[x].dev = dev; 135322150Shselasky /* store number of pages left from the array */ 136322150Shselasky fwp[x].numpages = num - x; 137322150Shselasky 138322150Shselasky /* allocate memory */ 139322150Shselasky err = bus_dmamem_alloc(dev->cmd.dma_tag, &fwp[x].virt_addr, 140322150Shselasky BUS_DMA_WAITOK | BUS_DMA_COHERENT, &fwp[x].dma_map); 141322150Shselasky if (err != 0) 142322150Shselasky goto failure; 143322150Shselasky 144322150Shselasky /* load memory into DMA */ 145322150Shselasky MLX5_DMA_LOCK(dev); 146322150Shselasky err = bus_dmamap_load( 147322150Shselasky dev->cmd.dma_tag, fwp[x].dma_map, fwp[x].virt_addr, 148322150Shselasky MLX5_ADAPTER_PAGE_SIZE, &mlx5_fwp_load_mem_cb, 149322150Shselasky fwp + x, BUS_DMA_WAITOK | BUS_DMA_COHERENT); 150322150Shselasky 151322150Shselasky while (fwp[x].load_done == MLX5_LOAD_ST_NONE) 152322150Shselasky MLX5_DMA_WAIT(dev); 153322150Shselasky MLX5_DMA_UNLOCK(dev); 154322150Shselasky 155322150Shselasky /* check for error */ 156322150Shselasky if (fwp[x].load_done != MLX5_LOAD_ST_SUCCESS) { 157322150Shselasky bus_dmamem_free(dev->cmd.dma_tag, fwp[x].virt_addr, 158322150Shselasky fwp[x].dma_map); 159322150Shselasky goto failure; 160322150Shselasky } 161322150Shselasky } 162322150Shselasky sx_xunlock(&dev->cmd.dma_sx); 163322150Shselasky return (fwp); 164322150Shselasky 165322150Shselaskyfailure: 166322150Shselasky while (x--) { 167322150Shselasky bus_dmamap_unload(dev->cmd.dma_tag, fwp[x].dma_map); 168322150Shselasky bus_dmamem_free(dev->cmd.dma_tag, fwp[x].virt_addr, fwp[x].dma_map); 169322150Shselasky } 170322150Shselasky sx_xunlock(&dev->cmd.dma_sx); 171322150Shselasky return (NULL); 172322150Shselasky} 173322150Shselasky 174322150Shselaskyvoid 175322150Shselaskymlx5_fwp_free(struct mlx5_fw_page *fwp) 176322150Shselasky{ 177322150Shselasky struct mlx5_core_dev *dev; 178322150Shselasky unsigned num; 179322150Shselasky 180322150Shselasky /* be NULL safe */ 181322150Shselasky if (fwp == NULL) 182322150Shselasky return; 183322150Shselasky 184322150Shselasky /* check for special case */ 185322150Shselasky if (fwp->numpages == 0) { 186322150Shselasky kfree(fwp); 187322150Shselasky return; 188322150Shselasky } 189322150Shselasky 190322150Shselasky num = fwp->numpages; 191322150Shselasky dev = fwp->dev; 192322150Shselasky 193322150Shselasky while (num--) { 194322150Shselasky bus_dmamap_unload(dev->cmd.dma_tag, fwp[num].dma_map); 195322150Shselasky bus_dmamem_free(dev->cmd.dma_tag, fwp[num].virt_addr, fwp[num].dma_map); 196322150Shselasky } 197322150Shselasky 198322150Shselasky kfree(fwp); 199322150Shselasky} 200322150Shselasky 201322150Shselaskyu64 202322150Shselaskymlx5_fwp_get_dma(struct mlx5_fw_page *fwp, size_t offset) 203322150Shselasky{ 204322150Shselasky size_t index = (offset / MLX5_ADAPTER_PAGE_SIZE); 205322150Shselasky KASSERT(index < fwp->numpages, ("Invalid offset: %lld", (long long)offset)); 206322150Shselasky 207322150Shselasky return ((fwp + index)->dma_addr + (offset % MLX5_ADAPTER_PAGE_SIZE)); 208322150Shselasky} 209322150Shselasky 210322150Shselaskyvoid * 211322150Shselaskymlx5_fwp_get_virt(struct mlx5_fw_page *fwp, size_t offset) 212322150Shselasky{ 213322150Shselasky size_t index = (offset / MLX5_ADAPTER_PAGE_SIZE); 214322150Shselasky KASSERT(index < fwp->numpages, ("Invalid offset: %lld", (long long)offset)); 215322150Shselasky 216322150Shselasky return ((char *)(fwp + index)->virt_addr + (offset % MLX5_ADAPTER_PAGE_SIZE)); 217322150Shselasky} 218322150Shselasky 219322150Shselaskystatic int 220322150Shselaskymlx5_insert_fw_page_locked(struct mlx5_core_dev *dev, struct mlx5_fw_page *nfp) 221322150Shselasky{ 222290650Shselasky struct rb_root *root = &dev->priv.page_root; 223290650Shselasky struct rb_node **new = &root->rb_node; 224290650Shselasky struct rb_node *parent = NULL; 225322146Shselasky struct mlx5_fw_page *tfp; 226290650Shselasky 227290650Shselasky while (*new) { 228290650Shselasky parent = *new; 229322146Shselasky tfp = rb_entry(parent, struct mlx5_fw_page, rb_node); 230322150Shselasky if (tfp->dma_addr < nfp->dma_addr) 231290650Shselasky new = &parent->rb_left; 232322150Shselasky else if (tfp->dma_addr > nfp->dma_addr) 233290650Shselasky new = &parent->rb_right; 234290650Shselasky else 235322150Shselasky return (-EEXIST); 236290650Shselasky } 237290650Shselasky 238290650Shselasky rb_link_node(&nfp->rb_node, parent, new); 239290650Shselasky rb_insert_color(&nfp->rb_node, root); 240322150Shselasky return (0); 241290650Shselasky} 242290650Shselasky 243322150Shselaskystatic struct mlx5_fw_page * 244322150Shselaskymlx5_remove_fw_page_locked(struct mlx5_core_dev *dev, bus_addr_t addr) 245290650Shselasky{ 246290650Shselasky struct rb_root *root = &dev->priv.page_root; 247290650Shselasky struct rb_node *tmp = root->rb_node; 248322146Shselasky struct mlx5_fw_page *result = NULL; 249322146Shselasky struct mlx5_fw_page *tfp; 250290650Shselasky 251290650Shselasky while (tmp) { 252322146Shselasky tfp = rb_entry(tmp, struct mlx5_fw_page, rb_node); 253322150Shselasky if (tfp->dma_addr < addr) { 254290650Shselasky tmp = tmp->rb_left; 255322150Shselasky } else if (tfp->dma_addr > addr) { 256290650Shselasky tmp = tmp->rb_right; 257290650Shselasky } else { 258322150Shselasky rb_erase(&tfp->rb_node, &dev->priv.page_root); 259290650Shselasky result = tfp; 260290650Shselasky break; 261290650Shselasky } 262290650Shselasky } 263322150Shselasky return (result); 264322150Shselasky} 265290650Shselasky 266322150Shselaskystatic int 267322150Shselaskyalloc_4k(struct mlx5_core_dev *dev, u64 *addr, u16 func_id) 268322150Shselasky{ 269322150Shselasky struct mlx5_fw_page *fwp; 270322150Shselasky int err; 271322150Shselasky 272322150Shselasky fwp = mlx5_fwp_alloc(dev, GFP_KERNEL, 1); 273322150Shselasky if (fwp == NULL) 274322150Shselasky return (-ENOMEM); 275322150Shselasky 276322150Shselasky fwp->func_id = func_id; 277322150Shselasky 278322150Shselasky MLX5_DMA_LOCK(dev); 279322150Shselasky err = mlx5_insert_fw_page_locked(dev, fwp); 280322150Shselasky MLX5_DMA_UNLOCK(dev); 281322150Shselasky 282322150Shselasky if (err != 0) { 283322150Shselasky mlx5_fwp_free(fwp); 284322150Shselasky } else { 285322150Shselasky /* make sure cached data is cleaned */ 286322150Shselasky mlx5_fwp_invalidate(fwp); 287322150Shselasky 288322150Shselasky /* store DMA address */ 289322150Shselasky *addr = fwp->dma_addr; 290322150Shselasky } 291322150Shselasky return (err); 292290650Shselasky} 293290650Shselasky 294322150Shselaskystatic void 295322150Shselaskyfree_4k(struct mlx5_core_dev *dev, u64 addr) 296322150Shselasky{ 297322150Shselasky struct mlx5_fw_page *fwp; 298322150Shselasky 299322150Shselasky MLX5_DMA_LOCK(dev); 300322150Shselasky fwp = mlx5_remove_fw_page_locked(dev, addr); 301322150Shselasky MLX5_DMA_UNLOCK(dev); 302322150Shselasky 303322150Shselasky if (fwp == NULL) { 304322150Shselasky mlx5_core_warn(dev, "Cannot free 4K page at 0x%llx\n", (long long)addr); 305322150Shselasky return; 306322150Shselasky } 307322150Shselasky mlx5_fwp_free(fwp); 308322150Shselasky} 309322150Shselasky 310290650Shselaskystatic int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, 311290650Shselasky s32 *npages, int boot) 312290650Shselasky{ 313290650Shselasky u32 in[MLX5_ST_SZ_DW(query_pages_in)]; 314290650Shselasky u32 out[MLX5_ST_SZ_DW(query_pages_out)]; 315290650Shselasky int err; 316290650Shselasky 317290650Shselasky memset(in, 0, sizeof(in)); 318290650Shselasky 319290650Shselasky MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); 320290650Shselasky MLX5_SET(query_pages_in, in, op_mod, 321290650Shselasky boot ? MLX5_BOOT_PAGES : MLX5_INIT_PAGES); 322290650Shselasky 323290650Shselasky memset(out, 0, sizeof(out)); 324290650Shselasky err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out)); 325290650Shselasky if (err) 326290650Shselasky return err; 327290650Shselasky 328290650Shselasky *npages = MLX5_GET(query_pages_out, out, num_pages); 329290650Shselasky *func_id = MLX5_GET(query_pages_out, out, function_id); 330290650Shselasky 331290650Shselasky return 0; 332290650Shselasky} 333290650Shselasky 334290650Shselaskystatic int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, 335290650Shselasky int notify_fail) 336290650Shselasky{ 337290650Shselasky struct mlx5_manage_pages_inbox *in; 338290650Shselasky struct mlx5_manage_pages_outbox out; 339290650Shselasky struct mlx5_manage_pages_inbox *nin; 340290650Shselasky int inlen; 341290650Shselasky u64 addr; 342290650Shselasky int err; 343308675Shselasky int i = 0; 344290650Shselasky 345290650Shselasky inlen = sizeof(*in) + npages * sizeof(in->pas[0]); 346290650Shselasky in = mlx5_vzalloc(inlen); 347290650Shselasky if (!in) { 348290650Shselasky mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); 349308675Shselasky err = -ENOMEM; 350308675Shselasky goto out_alloc; 351290650Shselasky } 352290650Shselasky memset(&out, 0, sizeof(out)); 353290650Shselasky 354290650Shselasky for (i = 0; i < npages; i++) { 355322150Shselasky err = alloc_4k(dev, &addr, func_id); 356322150Shselasky if (err) 357322150Shselasky goto out_alloc; 358290650Shselasky in->pas[i] = cpu_to_be64(addr); 359290650Shselasky } 360290650Shselasky 361290650Shselasky in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 362290650Shselasky in->hdr.opmod = cpu_to_be16(MLX5_PAGES_GIVE); 363290650Shselasky in->func_id = cpu_to_be16(func_id); 364290650Shselasky in->num_entries = cpu_to_be32(npages); 365290650Shselasky err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out)); 366290650Shselasky if (err) { 367290650Shselasky mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", 368290650Shselasky func_id, npages, err); 369290650Shselasky goto out_alloc; 370290650Shselasky } 371290650Shselasky dev->priv.fw_pages += npages; 372322144Shselasky dev->priv.pages_per_func[func_id] += npages; 373290650Shselasky 374290650Shselasky if (out.hdr.status) { 375290650Shselasky err = mlx5_cmd_status_to_err(&out.hdr); 376290650Shselasky if (err) { 377290650Shselasky mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n", 378290650Shselasky func_id, npages, out.hdr.status); 379290650Shselasky goto out_alloc; 380290650Shselasky } 381290650Shselasky } 382290650Shselasky 383290650Shselasky mlx5_core_dbg(dev, "err %d\n", err); 384290650Shselasky 385290650Shselasky goto out_free; 386290650Shselasky 387290650Shselaskyout_alloc: 388290650Shselasky if (notify_fail) { 389290650Shselasky nin = kzalloc(sizeof(*nin), GFP_KERNEL); 390322150Shselasky if (!nin) 391322150Shselasky goto out_4k; 392322150Shselasky 393290650Shselasky memset(&out, 0, sizeof(out)); 394290650Shselasky nin->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 395290650Shselasky nin->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE); 396308675Shselasky nin->func_id = cpu_to_be16(func_id); 397290650Shselasky if (mlx5_cmd_exec(dev, nin, sizeof(*nin), &out, sizeof(out))) 398290650Shselasky mlx5_core_warn(dev, "page notify failed\n"); 399290650Shselasky kfree(nin); 400290650Shselasky } 401322150Shselasky 402322150Shselaskyout_4k: 403290650Shselasky for (i--; i >= 0; i--) 404290650Shselasky free_4k(dev, be64_to_cpu(in->pas[i])); 405290650Shselaskyout_free: 406290650Shselasky kvfree(in); 407290650Shselasky return err; 408290650Shselasky} 409290650Shselasky 410290650Shselaskystatic int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, 411290650Shselasky int *nclaimed) 412290650Shselasky{ 413290650Shselasky struct mlx5_manage_pages_inbox in; 414290650Shselasky struct mlx5_manage_pages_outbox *out; 415290650Shselasky int num_claimed; 416290650Shselasky int outlen; 417290650Shselasky u64 addr; 418290650Shselasky int err; 419290650Shselasky int i; 420290650Shselasky 421290650Shselasky if (nclaimed) 422290650Shselasky *nclaimed = 0; 423290650Shselasky 424290650Shselasky memset(&in, 0, sizeof(in)); 425290650Shselasky outlen = sizeof(*out) + npages * sizeof(out->pas[0]); 426290650Shselasky out = mlx5_vzalloc(outlen); 427290650Shselasky if (!out) 428290650Shselasky return -ENOMEM; 429290650Shselasky 430290650Shselasky in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 431290650Shselasky in.hdr.opmod = cpu_to_be16(MLX5_PAGES_TAKE); 432290650Shselasky in.func_id = cpu_to_be16(func_id); 433290650Shselasky in.num_entries = cpu_to_be32(npages); 434290650Shselasky mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); 435290650Shselasky err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen); 436290650Shselasky if (err) { 437290650Shselasky mlx5_core_err(dev, "failed reclaiming pages\n"); 438290650Shselasky goto out_free; 439290650Shselasky } 440290650Shselasky 441290650Shselasky if (out->hdr.status) { 442290650Shselasky err = mlx5_cmd_status_to_err(&out->hdr); 443290650Shselasky goto out_free; 444290650Shselasky } 445290650Shselasky 446290650Shselasky num_claimed = be32_to_cpu(out->num_entries); 447290650Shselasky if (nclaimed) 448290650Shselasky *nclaimed = num_claimed; 449290650Shselasky 450290650Shselasky dev->priv.fw_pages -= num_claimed; 451322144Shselasky dev->priv.pages_per_func[func_id] -= num_claimed; 452290650Shselasky for (i = 0; i < num_claimed; i++) { 453290650Shselasky addr = be64_to_cpu(out->pas[i]); 454290650Shselasky free_4k(dev, addr); 455290650Shselasky } 456290650Shselasky 457290650Shselaskyout_free: 458290650Shselasky kvfree(out); 459290650Shselasky return err; 460290650Shselasky} 461290650Shselasky 462290650Shselaskystatic void pages_work_handler(struct work_struct *work) 463290650Shselasky{ 464290650Shselasky struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); 465290650Shselasky struct mlx5_core_dev *dev = req->dev; 466290650Shselasky int err = 0; 467290650Shselasky 468290650Shselasky if (req->npages < 0) 469290650Shselasky err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); 470290650Shselasky else if (req->npages > 0) 471290650Shselasky err = give_pages(dev, req->func_id, req->npages, 1); 472290650Shselasky 473290650Shselasky if (err) 474290650Shselasky mlx5_core_warn(dev, "%s fail %d\n", 475290650Shselasky req->npages < 0 ? "reclaim" : "give", err); 476290650Shselasky 477290650Shselasky kfree(req); 478290650Shselasky} 479290650Shselasky 480290650Shselaskyvoid mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, 481290650Shselasky s32 npages) 482290650Shselasky{ 483290650Shselasky struct mlx5_pages_req *req; 484290650Shselasky 485290650Shselasky req = kzalloc(sizeof(*req), GFP_ATOMIC); 486290650Shselasky if (!req) { 487290650Shselasky mlx5_core_warn(dev, "failed to allocate pages request\n"); 488290650Shselasky return; 489290650Shselasky } 490290650Shselasky 491290650Shselasky req->dev = dev; 492290650Shselasky req->func_id = func_id; 493290650Shselasky req->npages = npages; 494290650Shselasky INIT_WORK(&req->work, pages_work_handler); 495290650Shselasky if (!queue_work(dev->priv.pg_wq, &req->work)) 496290650Shselasky mlx5_core_warn(dev, "failed to queue pages handler work\n"); 497290650Shselasky} 498290650Shselasky 499290650Shselaskyint mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) 500290650Shselasky{ 501290650Shselasky u16 uninitialized_var(func_id); 502290650Shselasky s32 uninitialized_var(npages); 503290650Shselasky int err; 504290650Shselasky 505290650Shselasky err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); 506290650Shselasky if (err) 507290650Shselasky return err; 508290650Shselasky 509290650Shselasky mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", 510290650Shselasky npages, boot ? "boot" : "init", func_id); 511290650Shselasky 512290650Shselasky return give_pages(dev, func_id, npages, 0); 513290650Shselasky} 514290650Shselasky 515290650Shselaskyenum { 516290650Shselasky MLX5_BLKS_FOR_RECLAIM_PAGES = 12 517290650Shselasky}; 518290650Shselasky 519322144Shselaskys64 mlx5_wait_for_reclaim_vfs_pages(struct mlx5_core_dev *dev) 520322144Shselasky{ 521322144Shselasky int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 522322144Shselasky s64 prevpages = 0; 523322144Shselasky s64 npages = 0; 524322144Shselasky 525322144Shselasky while (!time_after(jiffies, end)) { 526322144Shselasky /* exclude own function, VFs only */ 527322144Shselasky npages = dev->priv.fw_pages - dev->priv.pages_per_func[0]; 528322144Shselasky if (!npages) 529322144Shselasky break; 530322144Shselasky 531322144Shselasky if (npages != prevpages) 532322144Shselasky end = end + msecs_to_jiffies(100); 533322144Shselasky 534322144Shselasky prevpages = npages; 535322144Shselasky msleep(1); 536322144Shselasky } 537322144Shselasky 538322144Shselasky if (npages) 539322144Shselasky mlx5_core_warn(dev, "FW did not return all VFs pages, will cause to memory leak\n"); 540322144Shselasky 541322144Shselasky return -npages; 542322144Shselasky} 543322144Shselasky 544290650Shselaskystatic int optimal_reclaimed_pages(void) 545290650Shselasky{ 546290650Shselasky struct mlx5_cmd_prot_block *block; 547290650Shselasky struct mlx5_cmd_layout *lay; 548290650Shselasky int ret; 549290650Shselasky 550290650Shselasky ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - 551290650Shselasky sizeof(struct mlx5_manage_pages_outbox)) / 552290650Shselasky FIELD_SIZEOF(struct mlx5_manage_pages_outbox, pas[0]); 553290650Shselasky 554290650Shselasky return ret; 555290650Shselasky} 556290650Shselasky 557290650Shselaskyint mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) 558290650Shselasky{ 559290650Shselasky int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 560322146Shselasky struct mlx5_fw_page *fwp; 561290650Shselasky struct rb_node *p; 562290650Shselasky int nclaimed = 0; 563290650Shselasky int err; 564290650Shselasky 565290650Shselasky do { 566290650Shselasky p = rb_first(&dev->priv.page_root); 567290650Shselasky if (p) { 568322146Shselasky fwp = rb_entry(p, struct mlx5_fw_page, rb_node); 569322148Shselasky if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 570322148Shselasky --dev->priv.fw_pages; 571322150Shselasky free_4k(dev, fwp->dma_addr); 572322148Shselasky nclaimed = 1; 573322148Shselasky } else { 574322148Shselasky err = reclaim_pages(dev, fwp->func_id, 575322148Shselasky optimal_reclaimed_pages(), 576322148Shselasky &nclaimed); 577322148Shselasky if (err) { 578322148Shselasky mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", 579322148Shselasky err); 580322148Shselasky return err; 581322148Shselasky } 582290650Shselasky } 583322148Shselasky 584290650Shselasky if (nclaimed) 585290650Shselasky end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 586290650Shselasky } 587290650Shselasky if (time_after(jiffies, end)) { 588290650Shselasky mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); 589290650Shselasky break; 590290650Shselasky } 591290650Shselasky } while (p); 592290650Shselasky 593290650Shselasky return 0; 594290650Shselasky} 595290650Shselasky 596290650Shselaskyvoid mlx5_pagealloc_init(struct mlx5_core_dev *dev) 597290650Shselasky{ 598322150Shselasky 599290650Shselasky dev->priv.page_root = RB_ROOT; 600290650Shselasky} 601290650Shselasky 602290650Shselaskyvoid mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) 603290650Shselasky{ 604290650Shselasky /* nothing */ 605290650Shselasky} 606290650Shselasky 607290650Shselaskyint mlx5_pagealloc_start(struct mlx5_core_dev *dev) 608290650Shselasky{ 609290650Shselasky dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); 610290650Shselasky if (!dev->priv.pg_wq) 611290650Shselasky return -ENOMEM; 612290650Shselasky 613290650Shselasky return 0; 614290650Shselasky} 615290650Shselasky 616290650Shselaskyvoid mlx5_pagealloc_stop(struct mlx5_core_dev *dev) 617290650Shselasky{ 618290650Shselasky destroy_workqueue(dev->priv.pg_wq); 619290650Shselasky} 620