tavor_rsrc.c revision 9517:b4839b0aa7a4
167754Smsmith/* 267754Smsmith * CDDL HEADER START 367754Smsmith * 4138287Smarks * The contents of this file are subject to the terms of the 567754Smsmith * Common Development and Distribution License (the "License"). 667754Smsmith * You may not use this file except in compliance with the License. 767754Smsmith * 867754Smsmith * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 967754Smsmith * or http://www.opensolaris.org/os/licensing. 1067754Smsmith * See the License for the specific language governing permissions 1167754Smsmith * and limitations under the License. 12126372Snjl * 1370243Smsmith * When distributing Covered Code, include this CDDL HEADER in each 1467754Smsmith * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1567754Smsmith * If applicable, add the following below this CDDL HEADER, with the 1667754Smsmith * fields enclosed by brackets "[]" replaced with your own identifying 1767754Smsmith * information: Portions Copyright [yyyy] [name of copyright owner] 1867754Smsmith * 1967754Smsmith * CDDL HEADER END 2067754Smsmith */ 2167754Smsmith 2267754Smsmith/* 2367754Smsmith * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 2467754Smsmith * Use is subject to license terms. 2567754Smsmith */ 2667754Smsmith 2767754Smsmith/* 2867754Smsmith * tavor_rsrc.c 2967754Smsmith * Tavor Resource Management Routines 3067754Smsmith * 3167754Smsmith * Implements all the routines necessary for setup, teardown, and 3267754Smsmith * alloc/free of all Tavor resources, including those that are managed 3367754Smsmith * by Tavor hardware or which live in Tavor's direct attached DDR memory. 3467754Smsmith */ 3567754Smsmith 3667754Smsmith#include <sys/types.h> 3767754Smsmith#include <sys/conf.h> 3867754Smsmith#include <sys/ddi.h> 3967754Smsmith#include <sys/sunddi.h> 4067754Smsmith#include <sys/modctl.h> 4167754Smsmith#include <sys/vmem.h> 4267754Smsmith#include <sys/bitmap.h> 4367754Smsmith 4467754Smsmith#include <sys/ib/adapters/tavor/tavor.h> 4567754Smsmith 4667754Smsmith/* 4767754Smsmith * The following routines are used for initializing and destroying 4867754Smsmith * the resource pools used by the Tavor resource allocation routines. 4967754Smsmith * They consist of four classes of object: 5067754Smsmith * 5167754Smsmith * Mailboxes: The "In" and "Out" mailbox types are used by the Tavor 5267754Smsmith * command interface routines. Mailboxes are used to pass information 5367754Smsmith * back and forth to the Tavor firmware. Either type of mailbox may 5467754Smsmith * be allocated from Tavor's direct attached DDR memory or from system 5567754Smsmith * memory (although currently all "In" mailboxes are in DDR and all "out" 5667754Smsmith * mailboxes come from system memory. 5767754Smsmith * 5867754Smsmith * HW entry objects: These objects represent resources required by the Tavor 5967754Smsmith * hardware. These objects include things like Queue Pair contexts (QPC), 6067754Smsmith * Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for 6167754Smsmith * supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory 6267754Smsmith * Protection Table entries (MPT), Memory Translation Table entries (MTT). 6367754Smsmith * 6467754Smsmith * What these objects all have in common is that they are each required 6567754Smsmith * to come from DDR memory, they are always allocated from tables, and 6667754Smsmith * they are not to be directly accessed (read or written) by driver 6767754Smsmith * software. 6867754Smsmith * One notable exceptions to this rule are the Extended QP contexts (EQPC), 6967754Smsmith * and the UAR scratch area (UAR_SCR), both of which are not directly 7067754Smsmith * accessible through the Tavor resource allocation routines, but both 7167754Smsmith * of which are also required to reside in DDR memory and are not to be 7267754Smsmith * manipulated by driver software (they are privately managed by Tavor 7367754Smsmith * hardware). 7467754Smsmith * The other notable exceptions are the UAR pages (UAR_PG) which are 7567754Smsmith * allocated from the UAR address space rather than DDR, and the UD 7667754Smsmith * address vectors (UDAV) which are similar to the common object types 7767754Smsmith * with the major difference being that UDAVs _are_ directly read and 7867754Smsmith * written by driver software. 7967754Smsmith * 8067754Smsmith * SW handle objects: These objects represent resources required by Tavor 8167754Smsmith * driver software. They are primarily software tracking structures, 8267754Smsmith * which are allocated from system memory (using kmem_cache). Several of 8367754Smsmith * the objects have both a "constructor" and "destructor" method 8467754Smsmith * associated with them (see below). 8567754Smsmith * 8667754Smsmith * Protection Domain (PD) handle objects: These objects are very much like 8767754Smsmith * a SW handle object with the notable difference that all PD handle 8867754Smsmith * objects have an actual Protection Domain number (PD) associated with 8967754Smsmith * them (and the PD number is allocated/managed through a separate 9067754Smsmith * vmem_arena specifically set aside for this purpose. 9167754Smsmith */ 9267754Smsmith 9367754Smsmithstatic int tavor_rsrc_mbox_init(tavor_state_t *state, 9467754Smsmith tavor_rsrc_mbox_info_t *info); 9567754Smsmithstatic void tavor_rsrc_mbox_fini(tavor_state_t *state, 9667754Smsmith tavor_rsrc_mbox_info_t *info); 9767754Smsmith 9867754Smsmithstatic int tavor_rsrc_hw_entries_init(tavor_state_t *state, 9967754Smsmith tavor_rsrc_hw_entry_info_t *info); 10067754Smsmithstatic void tavor_rsrc_hw_entries_fini(tavor_state_t *state, 10167754Smsmith tavor_rsrc_hw_entry_info_t *info); 10267754Smsmith 10367754Smsmithstatic int tavor_rsrc_sw_handles_init(tavor_state_t *state, 10467754Smsmith tavor_rsrc_sw_hdl_info_t *info); 10567754Smsmithstatic void tavor_rsrc_sw_handles_fini(tavor_state_t *state, 10667754Smsmith tavor_rsrc_sw_hdl_info_t *info); 10767754Smsmith 10867754Smsmithstatic int tavor_rsrc_pd_handles_init(tavor_state_t *state, 10967754Smsmith tavor_rsrc_sw_hdl_info_t *info); 11067754Smsmithstatic void tavor_rsrc_pd_handles_fini(tavor_state_t *state, 11167754Smsmith tavor_rsrc_sw_hdl_info_t *info); 11267754Smsmith 11367754Smsmith/* 11467754Smsmith * The following routines are used for allocating and freeing the specific 11567754Smsmith * types of objects described above from their associated resource pools. 11667754Smsmith */ 11767754Smsmithstatic int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, 11867754Smsmith uint_t num, tavor_rsrc_t *hdl); 11967754Smsmithstatic void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, 12067754Smsmith tavor_rsrc_t *hdl); 12167754Smsmith 12267754Smsmithstatic int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, 12380062Smsmith uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle, 12467754Smsmith uint_t sleepflag, tavor_rsrc_t *hdl); 12577424Smsmithstatic void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, 12691116Smsmith tavor_rsrc_t *hdl); 12767754Smsmith 12867754Smsmithstatic int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, 12967754Smsmith uint_t sleepflag, tavor_rsrc_t *hdl); 13067754Smsmithstatic void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, 13167754Smsmith tavor_rsrc_t *hdl); 13267754Smsmith 13367754Smsmithstatic int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, 13467754Smsmith uint_t sleepflag, tavor_rsrc_t *hdl); 13567754Smsmithstatic void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, 13667754Smsmith tavor_rsrc_t *hdl); 13767754Smsmith 13867754Smsmith/* 13967754Smsmith * The following routines are the constructors and destructors for several 14067754Smsmith * of the SW handle type objects. For certain types of SW handles objects 14167754Smsmith * (all of which are implemented using kmem_cache), we need to do some 14267754Smsmith * special field initialization (specifically, mutex_init/destroy). These 14367754Smsmith * routines enable that init and teardown. 14467754Smsmith */ 14567754Smsmithstatic int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags); 14667754Smsmithstatic void tavor_rsrc_pdhdl_destructor(void *pd, void *state); 14767754Smsmithstatic int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags); 14867754Smsmithstatic void tavor_rsrc_cqhdl_destructor(void *cq, void *state); 14967754Smsmithstatic int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags); 15091116Smsmithstatic void tavor_rsrc_qphdl_destructor(void *cq, void *state); 15167754Smsmithstatic int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags); 15267754Smsmithstatic void tavor_rsrc_srqhdl_destructor(void *srq, void *state); 15382367Smsmithstatic int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags); 15482367Smsmithstatic void tavor_rsrc_refcnt_destructor(void *rc, void *state); 155138287Smarksstatic int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags); 156138287Smarksstatic void tavor_rsrc_ahhdl_destructor(void *ah, void *state); 15767754Smsmithstatic int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags); 15899146Siwasakistatic void tavor_rsrc_mrhdl_destructor(void *mr, void *state); 15967754Smsmith 16099146Siwasaki/* 16167754Smsmith * Special routine to calculate and return the size of a MCG object based 16267754Smsmith * on current driver configuration (specifically, the number of QP per MCG 16367754Smsmith * that has been configured. 16467754Smsmith */ 16567754Smsmithstatic int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, 16667754Smsmith uint_t *mcg_size_shift); 16767754Smsmith 16867754Smsmith 169102550Siwasaki/* 17099679Siwasaki * tavor_rsrc_alloc() 17167754Smsmith * 17267754Smsmith * Context: Can be called from interrupt or base context. 173114237Snjl * The "sleepflag" parameter is used by all object allocators to 17499679Siwasaki * determine whether to SLEEP for resources or not. 175102550Siwasaki */ 176102550Siwasakiint 177102550Siwasakitavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num, 17899146Siwasaki uint_t sleepflag, tavor_rsrc_t **hdl) 179114237Snjl{ 18082367Smsmith tavor_rsrc_pool_info_t *rsrc_pool; 18199679Siwasaki tavor_rsrc_t *tmp_rsrc_hdl; 18282367Smsmith int flag, status = DDI_FAILURE; 18399679Siwasaki 18467754Smsmith TAVOR_TNF_ENTER(tavor_rsrc_alloc); 18567754Smsmith 18667754Smsmith ASSERT(state != NULL); 18767754Smsmith ASSERT(hdl != NULL); 18867754Smsmith 18977424Smsmith rsrc_pool = &state->ts_rsrc_hdl[rsrc]; 19067754Smsmith ASSERT(rsrc_pool != NULL); 19167754Smsmith 19267754Smsmith /* 19367754Smsmith * Allocate space for the object used to track the resource handle 19467754Smsmith */ 19567754Smsmith flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 19667754Smsmith tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache, 19767754Smsmith flag); 19867754Smsmith if (tmp_rsrc_hdl == NULL) { 19967754Smsmith TNF_PROBE_0(tavor_rsrc_alloc_kmca_fail, TAVOR_TNF_ERROR, ""); 20077424Smsmith TAVOR_TNF_EXIT(tavor_rsrc_alloc); 20177424Smsmith return (DDI_FAILURE); 20267754Smsmith } 20377424Smsmith _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl)) 20467754Smsmith 20567754Smsmith /* 20667754Smsmith * Set rsrc_hdl type. This is later used by the tavor_rsrc_free call 20773561Smsmith * to know what type of resource is being freed. 20867754Smsmith */ 20967754Smsmith tmp_rsrc_hdl->rsrc_type = rsrc; 21067754Smsmith 21167754Smsmith /* 21267754Smsmith * Depending on resource type, call the appropriate alloc routine 21391116Smsmith */ 21467754Smsmith switch (rsrc_pool->rsrc_type) { 21567754Smsmith case TAVOR_IN_MBOX: 21699146Siwasaki case TAVOR_OUT_MBOX: 21799146Siwasaki case TAVOR_INTR_IN_MBOX: 21867754Smsmith case TAVOR_INTR_OUT_MBOX: 21967754Smsmith status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl); 22067754Smsmith break; 22167754Smsmith 222138287Smarks case TAVOR_QPC: 223138287Smarks case TAVOR_CQC: 22467754Smsmith case TAVOR_SRQC: 225123315Snjl case TAVOR_EQC: 226123315Snjl case TAVOR_RDB: 227123315Snjl /* 228123315Snjl * Because these objects are NOT accessed by Tavor driver 229123315Snjl * software, we set the acc_handle parameter to zero. But 23067754Smsmith * if they are allocated in multiples, we specify here that 231129684Snjl * they must be aligned on a more restrictive boundary. 23267754Smsmith */ 233123315Snjl status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0, 234123315Snjl sleepflag, tmp_rsrc_hdl); 235123315Snjl break; 236123315Snjl 237123315Snjl case TAVOR_MPT: 23867754Smsmith /* 23967754Smsmith * Because these MPT objects are sometimes accessed by Tavor 240102550Siwasaki * driver software (FMR), we set the acc_handle parameter. But 24199679Siwasaki * if they are allocated in multiples, we specify here that 24267754Smsmith * they must be aligned on a more restrictive boundary. 24367754Smsmith */ 244114237Snjl status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 24599679Siwasaki state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl); 24669746Smsmith break; 24767754Smsmith 24867754Smsmith case TAVOR_MCG: 24967754Smsmith /* 25067754Smsmith * Tavor MCG entries are also NOT accessed by Tavor driver 25167754Smsmith * software, but because MCG entries do not have the same 25267754Smsmith * alignnment restrictions we loosen the constraint here. 25367754Smsmith */ 25467754Smsmith status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0, 25567754Smsmith sleepflag, tmp_rsrc_hdl); 25667754Smsmith break; 25767754Smsmith 25867754Smsmith case TAVOR_MTT: 25967754Smsmith case TAVOR_UDAV: 26067754Smsmith /* 26167754Smsmith * Because MTT segments are among the few HW resources that 26267754Smsmith * may be allocated in odd numbers, we specify a less 26367754Smsmith * restrictive alignment than for the above resources. 26467754Smsmith * 26567754Smsmith * Also because both UDAV and MTT segment objects are read 26667754Smsmith * and/or written by Tavor driver software, we set the 26767754Smsmith * acc_handle parameter to point to the ddi_acc_handle_t for 26867754Smsmith * the Tavor DDR memory. 26967754Smsmith */ 27067754Smsmith status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 27167754Smsmith state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl); 27267754Smsmith break; 27367754Smsmith 27467754Smsmith case TAVOR_UARPG: 27567754Smsmith /* 27667754Smsmith * Because UAR pages are written by Tavor driver software (for 27767754Smsmith * doorbells), we set the acc_handle parameter to point to 27867754Smsmith * the ddi_acc_handle_t for the Tavor UAR memory. 27991116Smsmith */ 28067754Smsmith status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 28167754Smsmith state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl); 28267754Smsmith break; 28367754Smsmith 28467754Smsmith case TAVOR_MRHDL: 28567754Smsmith case TAVOR_EQHDL: 28691116Smsmith case TAVOR_CQHDL: 28782367Smsmith case TAVOR_SRQHDL: 28882367Smsmith case TAVOR_AHHDL: 28967754Smsmith case TAVOR_QPHDL: 29067754Smsmith case TAVOR_REFCNT: 29167754Smsmith status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag, 29267754Smsmith tmp_rsrc_hdl); 29367754Smsmith break; 29487031Smsmith 29567754Smsmith case TAVOR_PDHDL: 29667754Smsmith status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag, 29767754Smsmith tmp_rsrc_hdl); 29867754Smsmith break; 29967754Smsmith 30099146Siwasaki default: 30199146Siwasaki TAVOR_WARNING(state, "unexpected resource type in alloc"); 30299146Siwasaki TNF_PROBE_0(tavor_rsrc_alloc_inv_rsrctype_fail, 30380062Smsmith TAVOR_TNF_ERROR, ""); 30499146Siwasaki break; 30599146Siwasaki } 30699146Siwasaki 30799146Siwasaki /* 30899146Siwasaki * If the resource allocation failed, then free the special resource 30999146Siwasaki * tracking structure and return failure. Otherwise return the 31099146Siwasaki * handle for the resource tracking structure. 31199146Siwasaki */ 31299146Siwasaki if (status != DDI_SUCCESS) { 31399146Siwasaki kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl); 31499146Siwasaki tmp_rsrc_hdl = NULL; 31599146Siwasaki TNF_PROBE_1(tavor_rsrc_alloc_fail, TAVOR_TNF_ERROR, "", 31699146Siwasaki tnf_uint, rsrc_type, rsrc_pool->rsrc_type); 31799146Siwasaki TAVOR_TNF_EXIT(tavor_rsrc_alloc); 31899146Siwasaki return (DDI_FAILURE); 31999146Siwasaki } else { 32099146Siwasaki *hdl = tmp_rsrc_hdl; 32199146Siwasaki TAVOR_TNF_EXIT(tavor_rsrc_alloc); 32299146Siwasaki return (DDI_SUCCESS); 32380062Smsmith } 32480062Smsmith} 32580062Smsmith 32699146Siwasaki 32799146Siwasaki/* 32899146Siwasaki * tavor_rsrc_free() 32999146Siwasaki * Context: Can be called from interrupt or base context. 33099146Siwasaki */ 33199146Siwasakivoid 33299146Siwasakitavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl) 33380062Smsmith{ 33480062Smsmith tavor_rsrc_pool_info_t *rsrc_pool; 33580062Smsmith 33680062Smsmith TAVOR_TNF_ENTER(tavor_rsrc_free); 33780062Smsmith 33880062Smsmith ASSERT(state != NULL); 33980062Smsmith ASSERT(hdl != NULL); 34080062Smsmith 34180062Smsmith rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type]; 34280062Smsmith ASSERT(rsrc_pool != NULL); 34399146Siwasaki 344138287Smarks /* 345138287Smarks * Depending on resource type, call the appropriate free routine 34699146Siwasaki */ 34767754Smsmith switch (rsrc_pool->rsrc_type) { 34867754Smsmith case TAVOR_IN_MBOX: 34967754Smsmith case TAVOR_OUT_MBOX: 35067754Smsmith case TAVOR_INTR_IN_MBOX: 35167754Smsmith case TAVOR_INTR_OUT_MBOX: 35267754Smsmith tavor_rsrc_mbox_free(rsrc_pool, *hdl); 35367754Smsmith break; 35467754Smsmith 35577424Smsmith case TAVOR_QPC: 35667754Smsmith case TAVOR_CQC: 35767754Smsmith case TAVOR_SRQC: 35877424Smsmith case TAVOR_EQC: 35999146Siwasaki case TAVOR_RDB: 36067754Smsmith case TAVOR_MCG: 36199146Siwasaki case TAVOR_MPT: 36299146Siwasaki case TAVOR_MTT: 36399146Siwasaki case TAVOR_UDAV: 36499146Siwasaki case TAVOR_UARPG: 36567754Smsmith tavor_rsrc_hw_entry_free(rsrc_pool, *hdl); 36667754Smsmith break; 36799146Siwasaki 36899146Siwasaki case TAVOR_MRHDL: 36999146Siwasaki case TAVOR_EQHDL: 37099146Siwasaki case TAVOR_CQHDL: 37167754Smsmith case TAVOR_SRQHDL: 37299679Siwasaki case TAVOR_AHHDL: 37399679Siwasaki case TAVOR_QPHDL: 37499679Siwasaki case TAVOR_REFCNT: 37599679Siwasaki tavor_rsrc_swhdl_free(rsrc_pool, *hdl); 37667754Smsmith break; 37767754Smsmith 37899146Siwasaki case TAVOR_PDHDL: 37999146Siwasaki tavor_rsrc_pdhdl_free(rsrc_pool, *hdl); 38099146Siwasaki break; 38199146Siwasaki 38299146Siwasaki default: 383123315Snjl TAVOR_WARNING(state, "unexpected resource type in free"); 384123315Snjl TNF_PROBE_0(tavor_rsrc_free_inv_rsrctype_fail, 38599146Siwasaki TAVOR_TNF_ERROR, ""); 38680062Smsmith break; 387138287Smarks } 388138287Smarks 389138287Smarks /* 390138287Smarks * Free the special resource tracking structure, set the handle to 391114237Snjl * NULL, and return. 39299146Siwasaki */ 393114237Snjl kmem_cache_free(state->ts_rsrc_cache, *hdl); 39499146Siwasaki *hdl = NULL; 39599146Siwasaki 39667754Smsmith TAVOR_TNF_EXIT(tavor_rsrc_free); 397138287Smarks} 398138287Smarks 39967754Smsmith 40080062Smsmith/* 40167754Smsmith * tavor_rsrc_init_phase1() 40267754Smsmith * 40367754Smsmith * Completes the first phase of Tavor resource/configuration init. 40467754Smsmith * This involves creating the kmem_cache for the "tavor_rsrc_t" 40577424Smsmith * structs, allocating the space for the resource pool handles, 40667754Smsmith * and setting up the "Out" mailboxes. 40767754Smsmith * 40867754Smsmith * When this function completes, the Tavor driver is ready to 40977424Smsmith * post the following commands which return information only in the 41067754Smsmith * "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER 41167754Smsmith * If any of these commands are to be posted at this time, they must be 41267754Smsmith * done so only when "spinning" (as the outstanding command list and 41367754Smsmith * EQ setup code has not yet run) 41467754Smsmith * 41567754Smsmith * Context: Only called from attach() path context 41667754Smsmith */ 41777424Smsmithint 41867754Smsmithtavor_rsrc_init_phase1(tavor_state_t *state) 41967754Smsmith{ 42067754Smsmith tavor_rsrc_pool_info_t *rsrc_pool; 42167754Smsmith tavor_rsrc_mbox_info_t mbox_info; 42267754Smsmith tavor_rsrc_cleanup_level_t cleanup; 42367754Smsmith tavor_cfg_profile_t *cfgprof; 42467754Smsmith uint64_t num, size; 42567754Smsmith int status; 426129684Snjl char *errormsg, *rsrc_name; 427129684Snjl 428129684Snjl TAVOR_TNF_ENTER(tavor_rsrc_init_phase1); 42967754Smsmith 43067754Smsmith ASSERT(state != NULL); 43167754Smsmith 43291116Smsmith /* This is where Phase 1 of resource initialization begins */ 43367754Smsmith cleanup = TAVOR_RSRC_CLEANUP_LEVEL0; 43467754Smsmith 435129684Snjl /* Build kmem cache name from Tavor instance */ 436129684Snjl rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP); 437129684Snjl TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE); 438129684Snjl 439129684Snjl /* 44073561Smsmith * Create the kmem_cache for "tavor_rsrc_t" structures 441123315Snjl * (kmem_cache_create will SLEEP until successful) 44273561Smsmith */ 44373561Smsmith state->ts_rsrc_cache = kmem_cache_create(rsrc_name, 444123315Snjl sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 445129684Snjl 446123315Snjl /* 447129684Snjl * Allocate an array of tavor_rsrc_pool_info_t's (used in all 448129684Snjl * subsequent resource allocations) 449129684Snjl */ 45091116Smsmith state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES * 451123315Snjl sizeof (tavor_rsrc_pool_info_t), KM_SLEEP); 45291116Smsmith 45367754Smsmith cfgprof = state->ts_cfg_profile; 454138287Smarks 455138287Smarks /* 45667754Smsmith * Initialize the resource pool for "Out" mailboxes. Notice that 457123315Snjl * the number of "Out" mailboxes, their size, and their location 45867754Smsmith * (DDR or system memory) is configurable. By default, however, 45967754Smsmith * all "Out" mailboxes are located in system memory only (because 460123315Snjl * they are primarily read from and never written to) 46167754Smsmith */ 46267754Smsmith num = ((uint64_t)1 << cfgprof->cp_log_num_outmbox); 46367754Smsmith size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size); 46467754Smsmith rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX]; 465129684Snjl rsrc_pool->rsrc_type = TAVOR_OUT_MBOX; 466129684Snjl rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 467123315Snjl rsrc_pool->rsrc_pool_size = (size * num); 46867754Smsmith rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size; 46967754Smsmith rsrc_pool->rsrc_quantum = size; 470129684Snjl rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 471123315Snjl rsrc_pool->rsrc_state = state; 472123315Snjl TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM); 47371867Smsmith mbox_info.mbi_num = num; 474123315Snjl mbox_info.mbi_size = size; 475123315Snjl mbox_info.mbi_rsrcpool = rsrc_pool; 476123315Snjl mbox_info.mbi_rsrcname = rsrc_name; 477129684Snjl status = tavor_rsrc_mbox_init(state, &mbox_info); 47867754Smsmith if (status != DDI_SUCCESS) { 479123315Snjl tavor_rsrc_fini(state, cleanup); 480123315Snjl /* Set "status" and "errormsg" and goto failure */ 481123315Snjl TAVOR_TNF_FAIL(DDI_FAILURE, "out mailboxes"); 48267754Smsmith goto rsrcinitp1_fail; 483123315Snjl } 484123315Snjl cleanup = TAVOR_RSRC_CLEANUP_LEVEL1; 485123315Snjl 48667754Smsmith /* 487123315Snjl * Initialize the Tavor "Out" mailbox list. This step actually uses 488123315Snjl * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the 48967754Smsmith * "Out" mailboxes, bind them for DMA access, and arrange them into 49067754Smsmith * an easily accessed fast-allocation mechanism (see tavor_cmd.c 49167754Smsmith * for more details) 49267754Smsmith */ 49367754Smsmith status = tavor_outmbox_list_init(state); 494129684Snjl if (status != DDI_SUCCESS) { 495129684Snjl tavor_rsrc_fini(state, cleanup); 49699679Siwasaki /* Set "status" and "errormsg" and goto failure */ 49767754Smsmith TAVOR_TNF_FAIL(DDI_FAILURE, "out mailbox list"); 49899679Siwasaki goto rsrcinitp1_fail; 49977424Smsmith } 50099679Siwasaki cleanup = TAVOR_RSRC_CLEANUP_LEVEL2; 50199679Siwasaki 50299679Siwasaki /* 50367754Smsmith * Initialize the resource pool for interrupt "Out" mailboxes. Notice 504129684Snjl * that the number of interrupt "Out" mailboxes, their size, and their 505129684Snjl * location (DDR or system memory) is configurable. By default, 50671867Smsmith * however, all interrupt "Out" mailboxes are located in system memory 50799679Siwasaki * only (because they are primarily read from and never written to) 50899679Siwasaki */ 50967754Smsmith num = ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox); 51099679Siwasaki size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size); 511129684Snjl rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX]; 51299679Siwasaki rsrc_pool->rsrc_type = TAVOR_INTR_OUT_MBOX; 51367754Smsmith rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 51499679Siwasaki rsrc_pool->rsrc_pool_size = (size * num); 51567754Smsmith rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size; 51667754Smsmith rsrc_pool->rsrc_quantum = size; 51767754Smsmith rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 518140216Snjl rsrc_pool->rsrc_state = state; 519140216Snjl TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM); 520140216Snjl mbox_info.mbi_num = num; 521140216Snjl mbox_info.mbi_size = size; 522140216Snjl mbox_info.mbi_rsrcpool = rsrc_pool; 523140216Snjl mbox_info.mbi_rsrcname = rsrc_name; 524140216Snjl status = tavor_rsrc_mbox_init(state, &mbox_info); 52577424Smsmith if (status != DDI_SUCCESS) { 52671867Smsmith tavor_rsrc_fini(state, cleanup); 52767754Smsmith /* Set "status" and "errormsg" and goto failure */ 52867754Smsmith TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailboxes"); 52967754Smsmith goto rsrcinitp1_fail; 53099679Siwasaki } 53199679Siwasaki cleanup = TAVOR_RSRC_CLEANUP_LEVEL3; 53299679Siwasaki 53399679Siwasaki /* 534129684Snjl * Initialize the Tavor "Out" mailbox list. This step actually uses 53599679Siwasaki * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the 53699679Siwasaki * "Out" mailboxes, bind them for DMA access, and arrange them into 53799679Siwasaki * an easily accessed fast-allocation mechanism (see tavor_cmd.c 53867754Smsmith * for more details) 539 */ 540 status = tavor_intr_outmbox_list_init(state); 541 if (status != DDI_SUCCESS) { 542 tavor_rsrc_fini(state, cleanup); 543 /* Set "status" and "errormsg" and goto failure */ 544 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailbox list"); 545 goto rsrcinitp1_fail; 546 } 547 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE; 548 549 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 550 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1); 551 return (DDI_SUCCESS); 552 553rsrcinitp1_fail: 554 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 555 TNF_PROBE_1(tavor_rsrc_init_phase1_fail, TAVOR_TNF_ERROR, "", 556 tnf_string, msg, errormsg); 557 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1); 558 return (status); 559} 560 561 562/* 563 * tavor_rsrc_init_phase2() 564 * Context: Only called from attach() path context 565 */ 566int 567tavor_rsrc_init_phase2(tavor_state_t *state) 568{ 569 tavor_rsrc_sw_hdl_info_t hdl_info; 570 tavor_rsrc_hw_entry_info_t entry_info; 571 tavor_rsrc_mbox_info_t mbox_info; 572 tavor_rsrc_pool_info_t *rsrc_pool; 573 tavor_rsrc_cleanup_level_t cleanup; 574 tavor_cfg_profile_t *cfgprof; 575 uint64_t num, max, size, num_prealloc; 576 uint64_t ddr_size, fw_size; 577 uint_t mcg_size, mcg_size_shift; 578 uint_t uarscr_size, mttsegment_sz; 579 int status; 580 char *errormsg, *rsrc_name; 581 582 TAVOR_TNF_ENTER(tavor_rsrc_init_phase2); 583 584 ASSERT(state != NULL); 585 586 /* Phase 2 initialization begins where Phase 1 left off */ 587 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE; 588 589 /* 590 * Calculate the extent of the DDR size and portion of which that 591 * is already reserved for Tavor firmware. (Note: this information 592 * is available because the QUERY_DDR and QUERY_FW commands have 593 * been posted to Tavor firmware prior to calling this routine) 594 */ 595 ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1; 596 fw_size = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1; 597 598 /* Build the DDR vmem arena name from Tavor instance */ 599 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP); 600 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM); 601 602 /* 603 * Do a vmem_create for the entire DDR range (not including the 604 * portion consumed by Tavor firmware). This creates the vmem arena 605 * from which all other DDR objects (specifically, tables of HW 606 * entries) will be allocated. 607 */ 608 state->ts_ddrvmem = vmem_create(rsrc_name, 609 (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size), 610 sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP); 611 if (state->ts_ddrvmem == NULL) { 612 tavor_rsrc_fini(state, cleanup); 613 /* Set "status" and "errormsg" and goto failure */ 614 TAVOR_TNF_FAIL(DDI_FAILURE, "DDR vmem"); 615 goto rsrcinitp2_fail; 616 } 617 cleanup = TAVOR_RSRC_CLEANUP_LEVEL5; 618 619 /* 620 * Initialize the resource pools for all objects that exist in 621 * Tavor DDR memory. This includes ("In") mailboxes, context tables 622 * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects. 623 */ 624 cfgprof = state->ts_cfg_profile; 625 626 /* 627 * Initialize the resource pool for the MPT table entries. Notice 628 * that the number of MPTs is configurable. The configured value must 629 * be less that the maximum value (obtained from the QUERY_DEV_LIM 630 * command) or the initialization will fail. Note also that a certain 631 * number of MPTs must be set aside for Tavor firmware use. 632 */ 633 num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 634 max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 635 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt); 636 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT]; 637 rsrc_pool->rsrc_type = TAVOR_MPT; 638 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 639 rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num); 640 rsrc_pool->rsrc_shift = TAVOR_MPT_SIZE_SHIFT; 641 rsrc_pool->rsrc_quantum = TAVOR_MPT_SIZE; 642 rsrc_pool->rsrc_align = (TAVOR_MPT_SIZE * num); 643 rsrc_pool->rsrc_state = state; 644 rsrc_pool->rsrc_start = NULL; 645 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM); 646 entry_info.hwi_num = num; 647 entry_info.hwi_max = max; 648 entry_info.hwi_prealloc = num_prealloc; 649 entry_info.hwi_rsrcpool = rsrc_pool; 650 entry_info.hwi_rsrcname = rsrc_name; 651 status = tavor_rsrc_hw_entries_init(state, &entry_info); 652 if (status != DDI_SUCCESS) { 653 tavor_rsrc_fini(state, cleanup); 654 /* Set "status" and "errormsg" and goto failure */ 655 TAVOR_TNF_FAIL(DDI_FAILURE, "MPT table"); 656 goto rsrcinitp2_fail; 657 } 658 cleanup = TAVOR_RSRC_CLEANUP_LEVEL6; 659 660 /* 661 * Initialize the resource pool for the MTT table entries. Notice 662 * that the number of MTTs is configurable. The configured value must 663 * be less that the maximum value (obtained from the QUERY_DEV_LIM 664 * command) or the initialization will fail. Note also that a certain 665 * number of MTT segments must be set aside for Tavor firmware use. 666 */ 667 mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT); 668 num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg); 669 max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg); 670 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg); 671 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT]; 672 rsrc_pool->rsrc_type = TAVOR_MTT; 673 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 674 rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num); 675 rsrc_pool->rsrc_shift = TAVOR_MTT_SIZE_SHIFT; 676 rsrc_pool->rsrc_quantum = mttsegment_sz; 677 rsrc_pool->rsrc_align = (TAVOR_MTT_SIZE * num); 678 rsrc_pool->rsrc_state = state; 679 rsrc_pool->rsrc_start = NULL; 680 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM); 681 entry_info.hwi_num = num; 682 entry_info.hwi_max = max; 683 entry_info.hwi_prealloc = num_prealloc; 684 entry_info.hwi_rsrcpool = rsrc_pool; 685 entry_info.hwi_rsrcname = rsrc_name; 686 status = tavor_rsrc_hw_entries_init(state, &entry_info); 687 if (status != DDI_SUCCESS) { 688 tavor_rsrc_fini(state, cleanup); 689 /* Set "status" and "errormsg" and goto failure */ 690 TAVOR_TNF_FAIL(DDI_FAILURE, "MTT table"); 691 goto rsrcinitp2_fail; 692 } 693 cleanup = TAVOR_RSRC_CLEANUP_LEVEL7; 694 695 /* 696 * Initialize the resource pool for the QPC table entries. Notice 697 * that the number of QPs is configurable. The configured value must 698 * be less that the maximum value (obtained from the QUERY_DEV_LIM 699 * command) or the initialization will fail. Note also that a certain 700 * number of QP contexts must be set aside for Tavor firmware use. 701 */ 702 num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 703 max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 704 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp); 705 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC]; 706 rsrc_pool->rsrc_type = TAVOR_QPC; 707 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 708 rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num); 709 rsrc_pool->rsrc_shift = TAVOR_QPC_SIZE_SHIFT; 710 rsrc_pool->rsrc_quantum = TAVOR_QPC_SIZE; 711 rsrc_pool->rsrc_align = (TAVOR_QPC_SIZE * num); 712 rsrc_pool->rsrc_state = state; 713 rsrc_pool->rsrc_start = NULL; 714 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM); 715 entry_info.hwi_num = num; 716 entry_info.hwi_max = max; 717 entry_info.hwi_prealloc = num_prealloc; 718 entry_info.hwi_rsrcpool = rsrc_pool; 719 entry_info.hwi_rsrcname = rsrc_name; 720 status = tavor_rsrc_hw_entries_init(state, &entry_info); 721 if (status != DDI_SUCCESS) { 722 tavor_rsrc_fini(state, cleanup); 723 /* Set "status" and "errormsg" and goto failure */ 724 TAVOR_TNF_FAIL(DDI_FAILURE, "QPC table"); 725 goto rsrcinitp2_fail; 726 } 727 cleanup = TAVOR_RSRC_CLEANUP_LEVEL8; 728 729 /* 730 * Initialize the resource pool for the RDB table entries. Notice 731 * that the number of RDBs is configurable. The configured value must 732 * be less that the maximum value (obtained from the QUERY_DEV_LIM 733 * command) or the initialization will fail. 734 */ 735 num = ((uint64_t)1 << cfgprof->cp_log_num_rdb); 736 max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob); 737 num_prealloc = 0; 738 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB]; 739 rsrc_pool->rsrc_type = TAVOR_RDB; 740 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 741 rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num); 742 rsrc_pool->rsrc_shift = TAVOR_RDB_SIZE_SHIFT; 743 rsrc_pool->rsrc_quantum = TAVOR_RDB_SIZE; 744 rsrc_pool->rsrc_align = (TAVOR_RDB_SIZE * num); 745 rsrc_pool->rsrc_state = state; 746 rsrc_pool->rsrc_start = NULL; 747 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM); 748 entry_info.hwi_num = num; 749 entry_info.hwi_max = max; 750 entry_info.hwi_prealloc = num_prealloc; 751 entry_info.hwi_rsrcpool = rsrc_pool; 752 entry_info.hwi_rsrcname = rsrc_name; 753 status = tavor_rsrc_hw_entries_init(state, &entry_info); 754 if (status != DDI_SUCCESS) { 755 tavor_rsrc_fini(state, cleanup); 756 /* Set "status" and "errormsg" and goto failure */ 757 TAVOR_TNF_FAIL(DDI_FAILURE, "RDB table"); 758 goto rsrcinitp2_fail; 759 } 760 cleanup = TAVOR_RSRC_CLEANUP_LEVEL9; 761 762 /* 763 * Initialize the resource pool for the CQC table entries. Notice 764 * that the number of CQs is configurable. The configured value must 765 * be less that the maximum value (obtained from the QUERY_DEV_LIM 766 * command) or the initialization will fail. Note also that a certain 767 * number of CQ contexts must be set aside for Tavor firmware use. 768 */ 769 num = ((uint64_t)1 << cfgprof->cp_log_num_cq); 770 max = ((uint64_t)1 << state->ts_devlim.log_max_cq); 771 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq); 772 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC]; 773 rsrc_pool->rsrc_type = TAVOR_CQC; 774 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 775 rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num); 776 rsrc_pool->rsrc_shift = TAVOR_CQC_SIZE_SHIFT; 777 rsrc_pool->rsrc_quantum = TAVOR_CQC_SIZE; 778 rsrc_pool->rsrc_align = (TAVOR_CQC_SIZE * num); 779 rsrc_pool->rsrc_state = state; 780 rsrc_pool->rsrc_start = NULL; 781 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM); 782 entry_info.hwi_num = num; 783 entry_info.hwi_max = max; 784 entry_info.hwi_prealloc = num_prealloc; 785 entry_info.hwi_rsrcpool = rsrc_pool; 786 entry_info.hwi_rsrcname = rsrc_name; 787 status = tavor_rsrc_hw_entries_init(state, &entry_info); 788 if (status != DDI_SUCCESS) { 789 tavor_rsrc_fini(state, cleanup); 790 /* Set "status" and "errormsg" and goto failure */ 791 TAVOR_TNF_FAIL(DDI_FAILURE, "CQC table"); 792 goto rsrcinitp2_fail; 793 } 794 cleanup = TAVOR_RSRC_CLEANUP_LEVEL10; 795 796 /* 797 * Initialize the resource pool for the Extended QPC table entries. 798 * Notice that the number of EQPCs must be the same as the number 799 * of QP contexts. So the initialization is constructed in a 800 * similar way as above (for TAVOR_QPC). One notable difference 801 * here, however, is that by setting the rsrc_quantum field to 802 * zero (indicating a zero-sized object) we indicate that the 803 * object is not allocatable. The EQPC table is, in fact, managed 804 * internally by the hardware and it is, therefore, unnecessary to 805 * initialize an additional vmem_arena for this type of object. 806 */ 807 num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 808 max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 809 num_prealloc = 0; 810 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 811 rsrc_pool->rsrc_type = TAVOR_EQPC; 812 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 813 rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num); 814 rsrc_pool->rsrc_shift = 0; 815 rsrc_pool->rsrc_quantum = 0; 816 rsrc_pool->rsrc_align = TAVOR_EQPC_SIZE; 817 rsrc_pool->rsrc_state = state; 818 rsrc_pool->rsrc_start = NULL; 819 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM); 820 entry_info.hwi_num = num; 821 entry_info.hwi_max = max; 822 entry_info.hwi_prealloc = num_prealloc; 823 entry_info.hwi_rsrcpool = rsrc_pool; 824 entry_info.hwi_rsrcname = rsrc_name; 825 status = tavor_rsrc_hw_entries_init(state, &entry_info); 826 if (status != DDI_SUCCESS) { 827 tavor_rsrc_fini(state, cleanup); 828 /* Set "status" and "errormsg" and goto failure */ 829 TAVOR_TNF_FAIL(DDI_FAILURE, "Extended QPC table"); 830 goto rsrcinitp2_fail; 831 } 832 cleanup = TAVOR_RSRC_CLEANUP_LEVEL11; 833 834 /* 835 * Initialize the resource pool for the UD address vector table 836 * entries. Notice that the number of UDAVs is configurable. The 837 * configured value must be less that the maximum value (obtained 838 * from the QUERY_DEV_LIM command) or the initialization will fail. 839 */ 840 num = ((uint64_t)1 << cfgprof->cp_log_num_ah); 841 max = ((uint64_t)1 << state->ts_devlim.log_max_av); 842 num_prealloc = 0; 843 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV]; 844 rsrc_pool->rsrc_type = TAVOR_UDAV; 845 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 846 rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num); 847 rsrc_pool->rsrc_shift = TAVOR_UDAV_SIZE_SHIFT; 848 rsrc_pool->rsrc_quantum = TAVOR_UDAV_SIZE; 849 rsrc_pool->rsrc_align = TAVOR_UDAV_SIZE; 850 rsrc_pool->rsrc_state = state; 851 rsrc_pool->rsrc_start = NULL; 852 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM); 853 entry_info.hwi_num = num; 854 entry_info.hwi_max = max; 855 entry_info.hwi_prealloc = num_prealloc; 856 entry_info.hwi_rsrcpool = rsrc_pool; 857 entry_info.hwi_rsrcname = rsrc_name; 858 status = tavor_rsrc_hw_entries_init(state, &entry_info); 859 if (status != DDI_SUCCESS) { 860 tavor_rsrc_fini(state, cleanup); 861 /* Set "status" and "errormsg" and goto failure */ 862 TAVOR_TNF_FAIL(DDI_FAILURE, "UDAV table"); 863 goto rsrcinitp2_fail; 864 } 865 cleanup = TAVOR_RSRC_CLEANUP_LEVEL12; 866 867 /* 868 * Initialize the resource pool for the UAR scratch table entries. 869 * Notice that the number of UARSCRs is configurable. The configured 870 * value must be less that the maximum value (obtained from the 871 * QUERY_DEV_LIM command) or the initialization will fail. 872 * Like the EQPCs above, UARSCR objects are not allocatable. The 873 * UARSCR table is also managed internally by the hardware and it 874 * is, therefore, unnecessary to initialize an additional vmem_arena 875 * for this type of object. We indicate this by setting the 876 * rsrc_quantum field to zero (indicating a zero-sized object). 877 */ 878 uarscr_size = state->ts_devlim.uarscr_entry_sz; 879 num = ((uint64_t)1 << cfgprof->cp_log_num_uar); 880 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 - 881 PAGESHIFT)); 882 num_prealloc = 0; 883 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 884 rsrc_pool->rsrc_type = TAVOR_UAR_SCR; 885 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 886 rsrc_pool->rsrc_pool_size = (uarscr_size * num); 887 rsrc_pool->rsrc_shift = 0; 888 rsrc_pool->rsrc_quantum = 0; 889 rsrc_pool->rsrc_align = uarscr_size; 890 rsrc_pool->rsrc_state = state; 891 rsrc_pool->rsrc_start = NULL; 892 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM); 893 entry_info.hwi_num = num; 894 entry_info.hwi_max = max; 895 entry_info.hwi_prealloc = num_prealloc; 896 entry_info.hwi_rsrcpool = rsrc_pool; 897 entry_info.hwi_rsrcname = rsrc_name; 898 status = tavor_rsrc_hw_entries_init(state, &entry_info); 899 if (status != DDI_SUCCESS) { 900 tavor_rsrc_fini(state, cleanup); 901 /* Set "status" and "errormsg" and goto failure */ 902 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR scratch table"); 903 goto rsrcinitp2_fail; 904 } 905 cleanup = TAVOR_RSRC_CLEANUP_LEVEL13; 906 907 /* 908 * Initialize the resource pool for the SRQC table entries. Notice 909 * that the number of SRQs is configurable. The configured value must 910 * be less that the maximum value (obtained from the QUERY_DEV_LIM 911 * command) or the initialization will fail. Note also that a certain 912 * number of SRQ contexts must be set aside for Tavor firmware use. 913 * 914 * Note: We only allocate these resources if SRQ is enabled in the 915 * config profile; see below. 916 */ 917 num = ((uint64_t)1 << cfgprof->cp_log_num_srq); 918 max = ((uint64_t)1 << state->ts_devlim.log_max_srq); 919 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq); 920 921 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC]; 922 rsrc_pool->rsrc_type = TAVOR_SRQC; 923 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 924 rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num); 925 rsrc_pool->rsrc_shift = TAVOR_SRQC_SIZE_SHIFT; 926 rsrc_pool->rsrc_quantum = TAVOR_SRQC_SIZE; 927 rsrc_pool->rsrc_align = (TAVOR_SRQC_SIZE * num); 928 rsrc_pool->rsrc_state = state; 929 rsrc_pool->rsrc_start = NULL; 930 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM); 931 entry_info.hwi_num = num; 932 entry_info.hwi_max = max; 933 entry_info.hwi_prealloc = num_prealloc; 934 entry_info.hwi_rsrcpool = rsrc_pool; 935 entry_info.hwi_rsrcname = rsrc_name; 936 937 /* 938 * SRQ support is configurable. Only if SRQ is enabled (the default) 939 * do we actually try to configure these resources. Otherwise, we 940 * simply set the cleanup level and continue on to the next resource 941 */ 942 if (state->ts_cfg_profile->cp_srq_enable != 0) { 943 status = tavor_rsrc_hw_entries_init(state, &entry_info); 944 if (status != DDI_SUCCESS) { 945 tavor_rsrc_fini(state, cleanup); 946 /* Set "status" and "errormsg" and goto failure */ 947 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQC table"); 948 goto rsrcinitp2_fail; 949 } 950 } 951 cleanup = TAVOR_RSRC_CLEANUP_LEVEL14; 952 953 /* 954 * Initialize the resource pool for "In" mailboxes. Notice that 955 * the number of "In" mailboxes, their size, and their location 956 * (DDR or system memory) is configurable. By default, however, 957 * all "In" mailboxes are located in system memory only (because 958 * they are primarily written to and rarely read from) 959 */ 960 num = ((uint64_t)1 << cfgprof->cp_log_num_inmbox); 961 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size); 962 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX]; 963 rsrc_pool->rsrc_type = TAVOR_IN_MBOX; 964 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 965 rsrc_pool->rsrc_pool_size = (size * num); 966 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size; 967 rsrc_pool->rsrc_quantum = size; 968 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 969 rsrc_pool->rsrc_state = state; 970 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM); 971 mbox_info.mbi_num = num; 972 mbox_info.mbi_size = size; 973 mbox_info.mbi_rsrcpool = rsrc_pool; 974 mbox_info.mbi_rsrcname = rsrc_name; 975 status = tavor_rsrc_mbox_init(state, &mbox_info); 976 if (status != DDI_SUCCESS) { 977 tavor_rsrc_fini(state, cleanup); 978 /* Set "status" and "errormsg" and goto failure */ 979 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailboxes"); 980 goto rsrcinitp2_fail; 981 } 982 cleanup = TAVOR_RSRC_CLEANUP_LEVEL15; 983 984 /* 985 * Initialize the Tavor "In" mailbox list. This step actually uses 986 * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the 987 * "In" mailboxes, bind them for DMA access, and arrange them into 988 * an easily accessed fast-allocation mechanism (see tavor_cmd.c 989 * for more details) 990 */ 991 status = tavor_inmbox_list_init(state); 992 if (status != DDI_SUCCESS) { 993 tavor_rsrc_fini(state, cleanup); 994 /* Set "status" and "errormsg" and goto failure */ 995 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailbox list"); 996 goto rsrcinitp2_fail; 997 } 998 cleanup = TAVOR_RSRC_CLEANUP_LEVEL16; 999 1000 /* 1001 * Initialize the resource pool for interrupt "In" mailboxes. Notice 1002 * that the number of interrupt "In" mailboxes, their size, and their 1003 * location (DDR or system memory) is configurable. By default, 1004 * however, all interrupt "In" mailboxes are located in system memory 1005 * only (because they are primarily written to and rarely read from) 1006 */ 1007 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox); 1008 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size); 1009 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX]; 1010 rsrc_pool->rsrc_type = TAVOR_INTR_IN_MBOX; 1011 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1012 rsrc_pool->rsrc_pool_size = (size * num); 1013 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size; 1014 rsrc_pool->rsrc_quantum = size; 1015 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN; 1016 rsrc_pool->rsrc_state = state; 1017 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM); 1018 mbox_info.mbi_num = num; 1019 mbox_info.mbi_size = size; 1020 mbox_info.mbi_rsrcpool = rsrc_pool; 1021 mbox_info.mbi_rsrcname = rsrc_name; 1022 status = tavor_rsrc_mbox_init(state, &mbox_info); 1023 if (status != DDI_SUCCESS) { 1024 tavor_rsrc_fini(state, cleanup); 1025 /* Set "status" and "errormsg" and goto failure */ 1026 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailboxes"); 1027 goto rsrcinitp2_fail; 1028 } 1029 cleanup = TAVOR_RSRC_CLEANUP_LEVEL17; 1030 1031 /* 1032 * Initialize the Tavor interrupt "In" mailbox list. This step 1033 * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to 1034 * preallocate the interrupt "In" mailboxes, bind them for DMA access, 1035 * and arrange them into an easily accessed fast-allocation mechanism 1036 * (see tavor_cmd.c for more details) 1037 */ 1038 status = tavor_intr_inmbox_list_init(state); 1039 if (status != DDI_SUCCESS) { 1040 tavor_rsrc_fini(state, cleanup); 1041 /* Set "status" and "errormsg" and goto failure */ 1042 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailbox list"); 1043 goto rsrcinitp2_fail; 1044 } 1045 cleanup = TAVOR_RSRC_CLEANUP_LEVEL18; 1046 1047 /* 1048 * Initialize the Tavor command handling interfaces. This step 1049 * sets up the outstanding command tracking mechanism for easy access 1050 * and fast allocation (see tavor_cmd.c for more details). 1051 */ 1052 status = tavor_outstanding_cmdlist_init(state); 1053 if (status != DDI_SUCCESS) { 1054 tavor_rsrc_fini(state, cleanup); 1055 /* Set "status" and "errormsg" and goto failure */ 1056 TAVOR_TNF_FAIL(DDI_FAILURE, "outstanding cmd list"); 1057 goto rsrcinitp2_fail; 1058 } 1059 cleanup = TAVOR_RSRC_CLEANUP_LEVEL19; 1060 1061 /* 1062 * Calculate (and validate) the size of Multicast Group (MCG) entries 1063 */ 1064 status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift); 1065 if (status != DDI_SUCCESS) { 1066 tavor_rsrc_fini(state, cleanup); 1067 /* Set "status" and "errormsg" and goto failure */ 1068 TAVOR_TNF_FAIL(DDI_FAILURE, "failed get MCG size"); 1069 goto rsrcinitp2_fail; 1070 } 1071 mcg_size = TAVOR_MCGMEM_SZ(state); 1072 1073 /* 1074 * Initialize the resource pool for the MCG table entries. Notice 1075 * that the number of MCGs is configurable. The configured value must 1076 * be less that the maximum value (obtained from the QUERY_DEV_LIM 1077 * command) or the initialization will fail. Note also that a certain 1078 * number of MCGs must be set aside for Tavor firmware use (they 1079 * correspond to the number of MCGs used by the internal hash 1080 * function. 1081 */ 1082 num = ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1083 max = ((uint64_t)1 << state->ts_devlim.log_max_mcg); 1084 num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash); 1085 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG]; 1086 rsrc_pool->rsrc_type = TAVOR_MCG; 1087 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1088 rsrc_pool->rsrc_pool_size = (mcg_size * num); 1089 rsrc_pool->rsrc_shift = mcg_size_shift; 1090 rsrc_pool->rsrc_quantum = mcg_size; 1091 rsrc_pool->rsrc_align = mcg_size; 1092 rsrc_pool->rsrc_state = state; 1093 rsrc_pool->rsrc_start = NULL; 1094 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM); 1095 entry_info.hwi_num = num; 1096 entry_info.hwi_max = max; 1097 entry_info.hwi_prealloc = num_prealloc; 1098 entry_info.hwi_rsrcpool = rsrc_pool; 1099 entry_info.hwi_rsrcname = rsrc_name; 1100 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1101 if (status != DDI_SUCCESS) { 1102 tavor_rsrc_fini(state, cleanup); 1103 /* Set "status" and "errormsg" and goto failure */ 1104 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG table"); 1105 goto rsrcinitp2_fail; 1106 } 1107 cleanup = TAVOR_RSRC_CLEANUP_LEVEL20; 1108 1109 /* 1110 * Initialize the resource pool for the EQC table entries. Notice 1111 * that the number of EQs is hardcoded. The hardcoded value should 1112 * be less that the maximum value (obtained from the QUERY_DEV_LIM 1113 * command) or the initialization will fail. 1114 */ 1115 num = TAVOR_NUM_EQ; 1116 max = ((uint64_t)1 << state->ts_devlim.log_max_eq); 1117 num_prealloc = 0; 1118 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC]; 1119 rsrc_pool->rsrc_type = TAVOR_EQC; 1120 rsrc_pool->rsrc_loc = TAVOR_IN_DDR; 1121 rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num); 1122 rsrc_pool->rsrc_shift = TAVOR_EQC_SIZE_SHIFT; 1123 rsrc_pool->rsrc_quantum = TAVOR_EQC_SIZE; 1124 rsrc_pool->rsrc_align = (TAVOR_EQC_SIZE * num); 1125 rsrc_pool->rsrc_state = state; 1126 rsrc_pool->rsrc_start = NULL; 1127 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM); 1128 entry_info.hwi_num = num; 1129 entry_info.hwi_max = max; 1130 entry_info.hwi_prealloc = num_prealloc; 1131 entry_info.hwi_rsrcpool = rsrc_pool; 1132 entry_info.hwi_rsrcname = rsrc_name; 1133 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1134 if (status != DDI_SUCCESS) { 1135 tavor_rsrc_fini(state, cleanup); 1136 /* Set "status" and "errormsg" and goto failure */ 1137 TAVOR_TNF_FAIL(DDI_FAILURE, "EQC table"); 1138 goto rsrcinitp2_fail; 1139 } 1140 cleanup = TAVOR_RSRC_CLEANUP_LEVEL21; 1141 1142 /* 1143 * Initialize the resource pools for all objects that exist in 1144 * system memory. This includes PD handles, MR handle, EQ handles, 1145 * QP handles, etc. These objects are almost entirely managed using 1146 * kmem_cache routines. (See comment above for more detail) 1147 */ 1148 1149 /* 1150 * Initialize the resource pool for the PD handles. Notice 1151 * that the number of PDHDLs is configurable. The configured value 1152 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1153 * command) or the initialization will fail. Note also that the PD 1154 * handle has constructor and destructor methods associated with it. 1155 */ 1156 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL]; 1157 rsrc_pool->rsrc_type = TAVOR_PDHDL; 1158 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1159 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_pd_s); 1160 rsrc_pool->rsrc_state = state; 1161 TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE); 1162 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd); 1163 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd); 1164 hdl_info.swi_rsrcpool = rsrc_pool; 1165 hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor; 1166 hdl_info.swi_destructor = tavor_rsrc_pdhdl_destructor; 1167 hdl_info.swi_rsrcname = rsrc_name; 1168 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1169 status = tavor_rsrc_pd_handles_init(state, &hdl_info); 1170 if (status != DDI_SUCCESS) { 1171 tavor_rsrc_fini(state, cleanup); 1172 /* Set "status" and "errormsg" and goto failure */ 1173 TAVOR_TNF_FAIL(DDI_FAILURE, "PD handle"); 1174 goto rsrcinitp2_fail; 1175 } 1176 cleanup = TAVOR_RSRC_CLEANUP_LEVEL22; 1177 1178 /* 1179 * Initialize the resource pool for the MR handles. Notice 1180 * that the number of MRHDLs is configurable. The configured value 1181 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1182 * command) or the initialization will fail. 1183 */ 1184 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL]; 1185 rsrc_pool->rsrc_type = TAVOR_MRHDL; 1186 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1187 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_mr_s); 1188 rsrc_pool->rsrc_state = state; 1189 TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE); 1190 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 1191 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 1192 hdl_info.swi_rsrcpool = rsrc_pool; 1193 hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor; 1194 hdl_info.swi_destructor = tavor_rsrc_mrhdl_destructor; 1195 hdl_info.swi_rsrcname = rsrc_name; 1196 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1197 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1198 if (status != DDI_SUCCESS) { 1199 tavor_rsrc_fini(state, cleanup); 1200 /* Set "status" and "errormsg" and goto failure */ 1201 TAVOR_TNF_FAIL(DDI_FAILURE, "MR handle"); 1202 goto rsrcinitp2_fail; 1203 } 1204 cleanup = TAVOR_RSRC_CLEANUP_LEVEL23; 1205 1206 /* 1207 * Initialize the resource pool for the EQ handles. Notice 1208 * that the number of EQHDLs is hardcoded. The hardcoded value 1209 * should be less that the maximum value (obtained from the 1210 * QUERY_DEV_LIM command) or the initialization will fail. 1211 */ 1212 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL]; 1213 rsrc_pool->rsrc_type = TAVOR_EQHDL; 1214 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1215 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_eq_s); 1216 rsrc_pool->rsrc_state = state; 1217 TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE); 1218 hdl_info.swi_num = TAVOR_NUM_EQ; 1219 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq); 1220 hdl_info.swi_rsrcpool = rsrc_pool; 1221 hdl_info.swi_constructor = NULL; 1222 hdl_info.swi_destructor = NULL; 1223 hdl_info.swi_rsrcname = rsrc_name; 1224 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1225 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1226 if (status != DDI_SUCCESS) { 1227 tavor_rsrc_fini(state, cleanup); 1228 /* Set "status" and "errormsg" and goto failure */ 1229 TAVOR_TNF_FAIL(DDI_FAILURE, "EQ handle"); 1230 goto rsrcinitp2_fail; 1231 } 1232 cleanup = TAVOR_RSRC_CLEANUP_LEVEL24; 1233 1234 /* 1235 * Initialize the resource pool for the CQ handles. Notice 1236 * that the number of CQHDLs is configurable. The configured value 1237 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1238 * command) or the initialization will fail. Note also that the CQ 1239 * handle has constructor and destructor methods associated with it. 1240 */ 1241 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL]; 1242 rsrc_pool->rsrc_type = TAVOR_CQHDL; 1243 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1244 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_cq_s); 1245 rsrc_pool->rsrc_state = state; 1246 TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE); 1247 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq); 1248 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq); 1249 hdl_info.swi_rsrcpool = rsrc_pool; 1250 hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor; 1251 hdl_info.swi_destructor = tavor_rsrc_cqhdl_destructor; 1252 hdl_info.swi_rsrcname = rsrc_name; 1253 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1254 TAVOR_SWHDL_TABLE_INIT); 1255 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t); 1256 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1257 if (status != DDI_SUCCESS) { 1258 tavor_rsrc_fini(state, cleanup); 1259 /* Set "status" and "errormsg" and goto failure */ 1260 TAVOR_TNF_FAIL(DDI_FAILURE, "CQ handle"); 1261 goto rsrcinitp2_fail; 1262 } 1263 1264 /* 1265 * Save away the pointer to the central list of CQ handle pointers 1266 * This this is used as a mechanism to enable fast CQnumber-to-CQhandle 1267 * lookup during EQ event processing. The table is a list of 1268 * tavor_cqhdl_t allocated by the above routine because of the 1269 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_cqhdl_t 1270 * as the number of CQs. 1271 */ 1272 state->ts_cqhdl = hdl_info.swi_table_ptr; 1273 cleanup = TAVOR_RSRC_CLEANUP_LEVEL25; 1274 1275 /* 1276 * Initialize the resource pool for the SRQ handles. Notice 1277 * that the number of SRQHDLs is configurable. The configured value 1278 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1279 * command) or the initialization will fail. Note also that the SRQ 1280 * handle has constructor and destructor methods associated with it. 1281 * 1282 * Note: We only allocate these resources if SRQ is enabled in the 1283 * config profile; see below. 1284 */ 1285 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL]; 1286 rsrc_pool->rsrc_type = TAVOR_SRQHDL; 1287 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1288 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_srq_s); 1289 rsrc_pool->rsrc_state = state; 1290 TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE); 1291 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq); 1292 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq); 1293 hdl_info.swi_rsrcpool = rsrc_pool; 1294 hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor; 1295 hdl_info.swi_destructor = tavor_rsrc_srqhdl_destructor; 1296 hdl_info.swi_rsrcname = rsrc_name; 1297 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1298 TAVOR_SWHDL_TABLE_INIT); 1299 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t); 1300 1301 /* 1302 * SRQ support is configurable. Only if SRQ is enabled (the default) 1303 * do we actually try to configure these resources. Otherwise, we 1304 * simply set the cleanup level and continue on to the next resource 1305 */ 1306 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1307 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1308 if (status != DDI_SUCCESS) { 1309 tavor_rsrc_fini(state, cleanup); 1310 /* Set "status" and "errormsg" and goto failure */ 1311 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQ handle"); 1312 goto rsrcinitp2_fail; 1313 } 1314 1315 /* 1316 * Save away the pointer to the central list of SRQ handle 1317 * pointers This this is used as a mechanism to enable fast 1318 * SRQnumber-to-SRQhandle lookup. The table is a list of 1319 * tavor_srqhdl_t allocated by the above routine because of the 1320 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many 1321 * tavor_srqhdl_t as the number of SRQs. 1322 */ 1323 state->ts_srqhdl = hdl_info.swi_table_ptr; 1324 } 1325 cleanup = TAVOR_RSRC_CLEANUP_LEVEL26; 1326 1327 /* 1328 * Initialize the resource pool for the address handles. Notice 1329 * that the number of AHHDLs is configurable. The configured value 1330 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1331 * command) or the initialization will fail. 1332 */ 1333 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL]; 1334 rsrc_pool->rsrc_type = TAVOR_AHHDL; 1335 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1336 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_ah_s); 1337 rsrc_pool->rsrc_state = state; 1338 TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE); 1339 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah); 1340 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av); 1341 hdl_info.swi_rsrcpool = rsrc_pool; 1342 hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor; 1343 hdl_info.swi_destructor = tavor_rsrc_ahhdl_destructor; 1344 hdl_info.swi_rsrcname = rsrc_name; 1345 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1346 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1347 if (status != DDI_SUCCESS) { 1348 tavor_rsrc_fini(state, cleanup); 1349 /* Set "status" and "errormsg" and goto failure */ 1350 TAVOR_TNF_FAIL(DDI_FAILURE, "AH handle"); 1351 goto rsrcinitp2_fail; 1352 } 1353 cleanup = TAVOR_RSRC_CLEANUP_LEVEL27; 1354 1355 /* 1356 * Initialize the resource pool for the QP handles. Notice 1357 * that the number of QPHDLs is configurable. The configured value 1358 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1359 * command) or the initialization will fail. Note also that the QP 1360 * handle has constructor and destructor methods associated with it. 1361 */ 1362 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL]; 1363 rsrc_pool->rsrc_type = TAVOR_QPHDL; 1364 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1365 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_qp_s); 1366 rsrc_pool->rsrc_state = state; 1367 TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE); 1368 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp); 1369 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp); 1370 hdl_info.swi_rsrcpool = rsrc_pool; 1371 hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor; 1372 hdl_info.swi_destructor = tavor_rsrc_qphdl_destructor; 1373 hdl_info.swi_rsrcname = rsrc_name; 1374 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT | 1375 TAVOR_SWHDL_TABLE_INIT); 1376 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t); 1377 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1378 if (status != DDI_SUCCESS) { 1379 tavor_rsrc_fini(state, cleanup); 1380 /* Set "status" and "errormsg" and goto failure */ 1381 TAVOR_TNF_FAIL(DDI_FAILURE, "QP handle"); 1382 goto rsrcinitp2_fail; 1383 } 1384 1385 /* 1386 * Save away the pointer to the central list of QP handle pointers 1387 * This this is used as a mechanism to enable fast QPnumber-to-QPhandle 1388 * lookup during CQ event processing. The table is a list of 1389 * tavor_qphdl_t allocated by the above routine because of the 1390 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_qphdl_t 1391 * as the number of QPs. 1392 */ 1393 state->ts_qphdl = hdl_info.swi_table_ptr; 1394 cleanup = TAVOR_RSRC_CLEANUP_LEVEL28; 1395 1396 /* 1397 * Initialize the resource pool for the reference count handles. 1398 * Notice that the number of REFCNTs is configurable, but it's value 1399 * is set to the number of MPTs. Since REFCNTs are used to support 1400 * shared memory regions, it is possible that we might require as 1401 * one REFCNT for every MPT. 1402 */ 1403 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT]; 1404 rsrc_pool->rsrc_type = TAVOR_REFCNT; 1405 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM; 1406 rsrc_pool->rsrc_quantum = sizeof (tavor_sw_refcnt_t); 1407 rsrc_pool->rsrc_state = state; 1408 TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE); 1409 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt); 1410 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt); 1411 hdl_info.swi_rsrcpool = rsrc_pool; 1412 hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor; 1413 hdl_info.swi_destructor = tavor_rsrc_refcnt_destructor; 1414 hdl_info.swi_rsrcname = rsrc_name; 1415 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT; 1416 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1417 if (status != DDI_SUCCESS) { 1418 tavor_rsrc_fini(state, cleanup); 1419 /* Set "status" and "errormsg" and goto failure */ 1420 TAVOR_TNF_FAIL(DDI_FAILURE, "reference count handle"); 1421 goto rsrcinitp2_fail; 1422 } 1423 cleanup = TAVOR_RSRC_CLEANUP_LEVEL29; 1424 1425 /* 1426 * Initialize the resource pool for the MCG handles. Notice that for 1427 * these MCG handles, we are allocating a table of structures (used to 1428 * keep track of the MCG entries that are being written to hardware 1429 * and to speed up multicast attach/detach operations). 1430 */ 1431 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1432 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg); 1433 hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT; 1434 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s); 1435 status = tavor_rsrc_sw_handles_init(state, &hdl_info); 1436 if (status != DDI_SUCCESS) { 1437 tavor_rsrc_fini(state, cleanup); 1438 /* Set "status" and "errormsg" and goto failure */ 1439 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG handle"); 1440 goto rsrcinitp2_fail; 1441 } 1442 state->ts_mcghdl = hdl_info.swi_table_ptr; 1443 cleanup = TAVOR_RSRC_CLEANUP_LEVEL30; 1444 1445 /* 1446 * Initialize the resource pools for all objects that exist in 1447 * UAR memory. The only objects that are allocated from UAR memory 1448 * are the UAR pages which are used for holding Tavor hardware's 1449 * doorbell registers. 1450 */ 1451 1452 /* 1453 * Initialize the resource pool for the UAR pages. Notice 1454 * that the number of UARPGs is configurable. The configured value 1455 * must be less that the maximum value (obtained from the QUERY_DEV_LIM 1456 * command) or the initialization will fail. Note also that by 1457 * specifying the rsrc_start parameter in advance, we direct the 1458 * initialization routine not to attempt to allocated space from the 1459 * Tavor DDR vmem_arena. 1460 */ 1461 num = ((uint64_t)1 << cfgprof->cp_log_num_uar); 1462 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 - 1463 PAGESHIFT)); 1464 num_prealloc = 0; 1465 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG]; 1466 rsrc_pool->rsrc_type = TAVOR_UARPG; 1467 rsrc_pool->rsrc_loc = TAVOR_IN_UAR; 1468 rsrc_pool->rsrc_pool_size = (num << PAGESHIFT); 1469 rsrc_pool->rsrc_shift = PAGESHIFT; 1470 rsrc_pool->rsrc_quantum = PAGESIZE; 1471 rsrc_pool->rsrc_align = PAGESIZE; 1472 rsrc_pool->rsrc_state = state; 1473 rsrc_pool->rsrc_start = (void *)state->ts_reg_uar_baseaddr; 1474 TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM); 1475 entry_info.hwi_num = num; 1476 entry_info.hwi_max = max; 1477 entry_info.hwi_prealloc = num_prealloc; 1478 entry_info.hwi_rsrcpool = rsrc_pool; 1479 entry_info.hwi_rsrcname = rsrc_name; 1480 status = tavor_rsrc_hw_entries_init(state, &entry_info); 1481 if (status != DDI_SUCCESS) { 1482 tavor_rsrc_fini(state, cleanup); 1483 /* Set "status" and "errormsg" and goto failure */ 1484 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR page table"); 1485 goto rsrcinitp2_fail; 1486 } 1487 cleanup = TAVOR_RSRC_CLEANUP_ALL; 1488 1489 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 1490 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2); 1491 return (DDI_SUCCESS); 1492 1493rsrcinitp2_fail: 1494 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN); 1495 TNF_PROBE_1(tavor_rsrc_init_phase2_fail, TAVOR_TNF_ERROR, "", 1496 tnf_string, msg, errormsg); 1497 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2); 1498 return (status); 1499} 1500 1501 1502/* 1503 * tavor_rsrc_fini() 1504 * Context: Only called from attach() and/or detach() path contexts 1505 */ 1506void 1507tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean) 1508{ 1509 tavor_rsrc_sw_hdl_info_t hdl_info; 1510 tavor_rsrc_hw_entry_info_t entry_info; 1511 tavor_rsrc_mbox_info_t mbox_info; 1512 tavor_cfg_profile_t *cfgprof; 1513 1514 TAVOR_TNF_ENTER(tavor_rsrc_fini); 1515 1516 ASSERT(state != NULL); 1517 1518 cfgprof = state->ts_cfg_profile; 1519 1520 switch (clean) { 1521 /* 1522 * If we add more resources that need to be cleaned up here, we should 1523 * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e. 1524 * corresponds to the last resource allocated). 1525 */ 1526 case TAVOR_RSRC_CLEANUP_ALL: 1527 /* Cleanup the UAR page resource pool */ 1528 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG]; 1529 tavor_rsrc_hw_entries_fini(state, &entry_info); 1530 /* FALLTHROUGH */ 1531 1532 case TAVOR_RSRC_CLEANUP_LEVEL30: 1533 /* Cleanup the central MCG handle pointers list */ 1534 hdl_info.swi_rsrcpool = NULL; 1535 hdl_info.swi_table_ptr = state->ts_mcghdl; 1536 hdl_info.swi_num = 1537 ((uint64_t)1 << cfgprof->cp_log_num_mcg); 1538 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s); 1539 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1540 /* FALLTHROUGH */ 1541 1542 case TAVOR_RSRC_CLEANUP_LEVEL29: 1543 /* Cleanup the reference count resource pool */ 1544 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_REFCNT]; 1545 hdl_info.swi_table_ptr = NULL; 1546 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1547 /* FALLTHROUGH */ 1548 1549 case TAVOR_RSRC_CLEANUP_LEVEL28: 1550 /* Cleanup the QP handle resource pool */ 1551 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPHDL]; 1552 hdl_info.swi_table_ptr = state->ts_qphdl; 1553 hdl_info.swi_num = 1554 ((uint64_t)1 << cfgprof->cp_log_num_qp); 1555 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t); 1556 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1557 /* FALLTHROUGH */ 1558 1559 case TAVOR_RSRC_CLEANUP_LEVEL27: 1560 /* Cleanup the address handle resource pool */ 1561 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_AHHDL]; 1562 hdl_info.swi_table_ptr = NULL; 1563 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1564 /* FALLTHROUGH */ 1565 1566 case TAVOR_RSRC_CLEANUP_LEVEL26: 1567 /* 1568 * Cleanup the SRQ handle resource pool. 1569 * 1570 * Note: We only clean up if SRQ is enabled. Otherwise we 1571 * simply fallthrough to the next resource cleanup. 1572 */ 1573 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1574 hdl_info.swi_rsrcpool = 1575 &state->ts_rsrc_hdl[TAVOR_SRQHDL]; 1576 hdl_info.swi_table_ptr = state->ts_srqhdl; 1577 hdl_info.swi_num = 1578 ((uint64_t)1 << cfgprof->cp_log_num_srq); 1579 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t); 1580 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1581 } 1582 /* FALLTHROUGH */ 1583 1584 case TAVOR_RSRC_CLEANUP_LEVEL25: 1585 /* Cleanup the CQ handle resource pool */ 1586 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQHDL]; 1587 hdl_info.swi_table_ptr = state->ts_cqhdl; 1588 hdl_info.swi_num = 1589 ((uint64_t)1 << cfgprof->cp_log_num_cq); 1590 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t); 1591 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1592 /* FALLTHROUGH */ 1593 1594 case TAVOR_RSRC_CLEANUP_LEVEL24: 1595 /* Cleanup the EQ handle resource pool */ 1596 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQHDL]; 1597 hdl_info.swi_table_ptr = NULL; 1598 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1599 /* FALLTHROUGH */ 1600 1601 case TAVOR_RSRC_CLEANUP_LEVEL23: 1602 /* Cleanup the MR handle resource pool */ 1603 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MRHDL]; 1604 hdl_info.swi_table_ptr = NULL; 1605 tavor_rsrc_sw_handles_fini(state, &hdl_info); 1606 /* FALLTHROUGH */ 1607 1608 case TAVOR_RSRC_CLEANUP_LEVEL22: 1609 /* Cleanup the PD handle resource pool */ 1610 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_PDHDL]; 1611 hdl_info.swi_table_ptr = NULL; 1612 tavor_rsrc_pd_handles_fini(state, &hdl_info); 1613 /* FALLTHROUGH */ 1614 1615 case TAVOR_RSRC_CLEANUP_LEVEL21: 1616 /* Cleanup the EQC table resource pool */ 1617 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC]; 1618 tavor_rsrc_hw_entries_fini(state, &entry_info); 1619 /* FALLTHROUGH */ 1620 1621 case TAVOR_RSRC_CLEANUP_LEVEL20: 1622 /* Cleanup the MCG table resource pool */ 1623 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG]; 1624 tavor_rsrc_hw_entries_fini(state, &entry_info); 1625 /* FALLTHROUGH */ 1626 1627 case TAVOR_RSRC_CLEANUP_LEVEL19: 1628 /* Cleanup the outstanding command list */ 1629 tavor_outstanding_cmdlist_fini(state); 1630 /* FALLTHROUGH */ 1631 1632 case TAVOR_RSRC_CLEANUP_LEVEL18: 1633 /* Cleanup the "In" mailbox list */ 1634 tavor_intr_inmbox_list_fini(state); 1635 /* FALLTHROUGH */ 1636 1637 case TAVOR_RSRC_CLEANUP_LEVEL17: 1638 /* Cleanup the interrupt "In" mailbox resource pool */ 1639 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[ 1640 TAVOR_INTR_IN_MBOX]; 1641 tavor_rsrc_mbox_fini(state, &mbox_info); 1642 /* FALLTHROUGH */ 1643 1644 case TAVOR_RSRC_CLEANUP_LEVEL16: 1645 /* Cleanup the "In" mailbox list */ 1646 tavor_inmbox_list_fini(state); 1647 /* FALLTHROUGH */ 1648 1649 case TAVOR_RSRC_CLEANUP_LEVEL15: 1650 /* Cleanup the "In" mailbox resource pool */ 1651 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX]; 1652 tavor_rsrc_mbox_fini(state, &mbox_info); 1653 /* FALLTHROUGH */ 1654 1655 case TAVOR_RSRC_CLEANUP_LEVEL14: 1656 /* 1657 * Cleanup the SRQC table resource pool. 1658 * 1659 * Note: We only clean up if SRQ is enabled. Otherwise we 1660 * simply fallthrough to the next resource cleanup. 1661 */ 1662 if (state->ts_cfg_profile->cp_srq_enable != 0) { 1663 entry_info.hwi_rsrcpool = 1664 &state->ts_rsrc_hdl[TAVOR_SRQC]; 1665 tavor_rsrc_hw_entries_fini(state, &entry_info); 1666 } 1667 /* FALLTHROUGH */ 1668 1669 case TAVOR_RSRC_CLEANUP_LEVEL13: 1670 /* Cleanup the UAR scratch table resource pool */ 1671 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 1672 tavor_rsrc_hw_entries_fini(state, &entry_info); 1673 /* FALLTHROUGH */ 1674 1675 case TAVOR_RSRC_CLEANUP_LEVEL12: 1676 /* Cleanup the UDAV table resource pool */ 1677 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV]; 1678 tavor_rsrc_hw_entries_fini(state, &entry_info); 1679 /* FALLTHROUGH */ 1680 1681 case TAVOR_RSRC_CLEANUP_LEVEL11: 1682 /* Cleanup the EQPC table resource pool */ 1683 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 1684 tavor_rsrc_hw_entries_fini(state, &entry_info); 1685 /* FALLTHROUGH */ 1686 1687 case TAVOR_RSRC_CLEANUP_LEVEL10: 1688 /* Cleanup the CQC table resource pool */ 1689 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC]; 1690 tavor_rsrc_hw_entries_fini(state, &entry_info); 1691 /* FALLTHROUGH */ 1692 1693 case TAVOR_RSRC_CLEANUP_LEVEL9: 1694 /* Cleanup the RDB table resource pool */ 1695 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB]; 1696 tavor_rsrc_hw_entries_fini(state, &entry_info); 1697 /* FALLTHROUGH */ 1698 1699 case TAVOR_RSRC_CLEANUP_LEVEL8: 1700 /* Cleanup the QPC table resource pool */ 1701 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC]; 1702 tavor_rsrc_hw_entries_fini(state, &entry_info); 1703 /* FALLTHROUGH */ 1704 1705 case TAVOR_RSRC_CLEANUP_LEVEL7: 1706 /* Cleanup the MTT table resource pool */ 1707 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT]; 1708 tavor_rsrc_hw_entries_fini(state, &entry_info); 1709 /* FALLTHROUGH */ 1710 1711 case TAVOR_RSRC_CLEANUP_LEVEL6: 1712 /* Cleanup the MPT table resource pool */ 1713 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT]; 1714 tavor_rsrc_hw_entries_fini(state, &entry_info); 1715 /* FALLTHROUGH */ 1716 1717 case TAVOR_RSRC_CLEANUP_LEVEL5: 1718 /* Destroy the vmem arena for DDR memory */ 1719 vmem_destroy(state->ts_ddrvmem); 1720 break; 1721 1722 /* 1723 * The cleanup below comes from the "Phase 1" initialization step. 1724 * (see tavor_rsrc_init_phase1() above) 1725 */ 1726 case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE: 1727 /* Cleanup the interrupt "Out" mailbox list */ 1728 tavor_intr_outmbox_list_fini(state); 1729 /* FALLTHROUGH */ 1730 1731 case TAVOR_RSRC_CLEANUP_LEVEL3: 1732 /* Cleanup the "Out" mailbox resource pool */ 1733 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[ 1734 TAVOR_INTR_OUT_MBOX]; 1735 tavor_rsrc_mbox_fini(state, &mbox_info); 1736 /* FALLTHROUGH */ 1737 1738 case TAVOR_RSRC_CLEANUP_LEVEL2: 1739 /* Cleanup the "Out" mailbox list */ 1740 tavor_outmbox_list_fini(state); 1741 /* FALLTHROUGH */ 1742 1743 case TAVOR_RSRC_CLEANUP_LEVEL1: 1744 /* Cleanup the "Out" mailbox resource pool */ 1745 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX]; 1746 tavor_rsrc_mbox_fini(state, &mbox_info); 1747 /* FALLTHROUGH */ 1748 1749 case TAVOR_RSRC_CLEANUP_LEVEL0: 1750 /* Free the array of tavor_rsrc_pool_info_t's */ 1751 kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES * 1752 sizeof (tavor_rsrc_pool_info_t)); 1753 kmem_cache_destroy(state->ts_rsrc_cache); 1754 break; 1755 1756 default: 1757 TAVOR_WARNING(state, "unexpected resource cleanup level"); 1758 TNF_PROBE_0(tavor_rsrc_fini_default_fail, TAVOR_TNF_ERROR, ""); 1759 TAVOR_TNF_EXIT(tavor_rsrc_fini); 1760 return; 1761 } 1762 1763 TAVOR_TNF_EXIT(tavor_rsrc_fini); 1764} 1765 1766 1767/* 1768 * tavor_rsrc_mbox_init() 1769 * Context: Only called from attach() path context 1770 */ 1771static int 1772tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info) 1773{ 1774 tavor_rsrc_pool_info_t *rsrc_pool; 1775 tavor_rsrc_priv_mbox_t *priv; 1776 vmem_t *vmp; 1777 uint64_t offset; 1778 uint_t dma_xfer_mode; 1779 1780 TAVOR_TNF_ENTER(tavor_rsrc_mbox_init); 1781 1782 ASSERT(state != NULL); 1783 ASSERT(info != NULL); 1784 1785 rsrc_pool = info->mbi_rsrcpool; 1786 ASSERT(rsrc_pool != NULL); 1787 1788 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent; 1789 1790 /* Allocate and initialize mailbox private structure */ 1791 priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP); 1792 priv->pmb_dip = state->ts_dip; 1793 priv->pmb_acchdl = state->ts_reg_ddrhdl; 1794 priv->pmb_devaccattr = state->ts_reg_accattr; 1795 priv->pmb_xfer_mode = dma_xfer_mode; 1796 1797 /* 1798 * Initialize many of the default DMA attributes. Then set alignment 1799 * and scatter-gather restrictions specific for mailbox memory. 1800 */ 1801 tavor_dma_attr_init(&priv->pmb_dmaattr); 1802 priv->pmb_dmaattr.dma_attr_align = TAVOR_MBOX_ALIGN; 1803 priv->pmb_dmaattr.dma_attr_sgllen = 1; 1804 1805 rsrc_pool->rsrc_private = priv; 1806 1807 /* Is object in DDR memory or system memory? */ 1808 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) { 1809 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem, 1810 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align, 1811 0, 0, NULL, NULL, VM_SLEEP); 1812 if (rsrc_pool->rsrc_ddr_offset == NULL) { 1813 /* Unable to alloc space for mailboxes */ 1814 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t)); 1815 TNF_PROBE_0(tavor_rsrc_mbox_init_vma_fail, 1816 TAVOR_TNF_ERROR, ""); 1817 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init); 1818 return (DDI_FAILURE); 1819 } 1820 1821 /* Calculate offset and starting point (in DDR) */ 1822 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset - 1823 state->ts_ddr.ddr_baseaddr); 1824 rsrc_pool->rsrc_start = 1825 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr + 1826 offset); 1827 1828 /* Create new vmem arena for the mailboxes */ 1829 vmp = vmem_create(info->mbi_rsrcname, 1830 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size, 1831 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 1832 if (vmp == NULL) { 1833 /* Unable to create vmem arena */ 1834 vmem_xfree(state->ts_ddrvmem, 1835 rsrc_pool->rsrc_ddr_offset, 1836 rsrc_pool->rsrc_pool_size); 1837 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t)); 1838 TNF_PROBE_0(tavor_rsrc_mbox_init_vmem_create_fail, 1839 TAVOR_TNF_ERROR, ""); 1840 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init); 1841 return (DDI_FAILURE); 1842 } 1843 rsrc_pool->rsrc_vmp = vmp; 1844 } else { 1845 rsrc_pool->rsrc_ddr_offset = NULL; 1846 rsrc_pool->rsrc_start = NULL; 1847 rsrc_pool->rsrc_vmp = NULL; 1848 } 1849 1850 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init); 1851 return (DDI_SUCCESS); 1852} 1853 1854 1855/* 1856 * tavor_rsrc_mbox_fini() 1857 * Context: Only called from attach() and/or detach() path contexts 1858 */ 1859static void 1860tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info) 1861{ 1862 tavor_rsrc_pool_info_t *rsrc_pool; 1863 1864 TAVOR_TNF_ENTER(tavor_rsrc_mbox_fini); 1865 1866 ASSERT(state != NULL); 1867 ASSERT(info != NULL); 1868 1869 rsrc_pool = info->mbi_rsrcpool; 1870 ASSERT(rsrc_pool != NULL); 1871 1872 /* If mailboxes are DDR memory, then destroy and free up vmem */ 1873 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) { 1874 1875 /* Destroy the specially created mbox vmem arena */ 1876 vmem_destroy(rsrc_pool->rsrc_vmp); 1877 1878 /* Free up the region from the ddr_vmem arena */ 1879 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset, 1880 rsrc_pool->rsrc_pool_size); 1881 } 1882 1883 /* Free up the private struct */ 1884 kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t)); 1885 1886 TAVOR_TNF_EXIT(tavor_rsrc_mbox_fini); 1887} 1888 1889 1890/* 1891 * tavor_rsrc_hw_entries_init() 1892 * Context: Only called from attach() path context 1893 */ 1894static int 1895tavor_rsrc_hw_entries_init(tavor_state_t *state, 1896 tavor_rsrc_hw_entry_info_t *info) 1897{ 1898 tavor_rsrc_pool_info_t *rsrc_pool; 1899 tavor_rsrc_t *rsvd_rsrc = NULL; 1900 vmem_t *vmp; 1901 uint64_t num_hwentry, max_hwentry, num_prealloc; 1902 uint64_t offset; 1903 int status; 1904 1905 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_init); 1906 1907 ASSERT(state != NULL); 1908 ASSERT(info != NULL); 1909 1910 rsrc_pool = info->hwi_rsrcpool; 1911 ASSERT(rsrc_pool != NULL); 1912 num_hwentry = info->hwi_num; 1913 max_hwentry = info->hwi_max; 1914 num_prealloc = info->hwi_prealloc; 1915 1916 /* Make sure number of HW entries makes sense */ 1917 if (num_hwentry > max_hwentry) { 1918 TNF_PROBE_2(tavor_rsrc_hw_entries_init_toomany_fail, 1919 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of HW " 1920 "entries exceeds device maximum", tnf_uint, maxhw, 1921 max_hwentry); 1922 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 1923 return (DDI_FAILURE); 1924 } 1925 1926 /* 1927 * Determine if we need to allocate DDR space to set up the 1928 * "rsrc_start" pointer. Not necessary if "rsrc_start" has already 1929 * been initialized (as is the case for the UAR page init). 1930 */ 1931 if (rsrc_pool->rsrc_start == NULL) { 1932 /* Make sure HW entries table is aligned as specified */ 1933 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem, 1934 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align, 1935 0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT); 1936 if (rsrc_pool->rsrc_ddr_offset == NULL) { 1937 /* Unable to alloc space for aligned HW table */ 1938 TNF_PROBE_0(tavor_rsrc_hw_entry_table_vmxalloc_fail, 1939 TAVOR_TNF_ERROR, ""); 1940 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 1941 return (DDI_FAILURE); 1942 } 1943 1944 /* Calculate offset and starting point (in DDR) */ 1945 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset - 1946 state->ts_ddr.ddr_baseaddr); 1947 rsrc_pool->rsrc_start = 1948 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr + 1949 offset); 1950 } else { 1951 rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start; 1952 } 1953 1954 /* 1955 * Create new vmem arena for the HW entries table (if rsrc_quantum 1956 * is non-zero). Otherwise if rsrc_quantum is zero, then these HW 1957 * entries are not going to be dynamically allocatable (i.e. they 1958 * won't be allocated/freed through tavor_rsrc_alloc/free). This 1959 * latter option is used for EQPC and UARSCR resource which are, in 1960 * fact, managed by the Tavor hardware. 1961 */ 1962 if (rsrc_pool->rsrc_quantum != 0) { 1963 vmp = vmem_create(info->hwi_rsrcname, 1964 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size, 1965 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP); 1966 if (vmp == NULL) { 1967 /* Unable to create vmem arena */ 1968 if (rsrc_pool->rsrc_ddr_offset != 1969 rsrc_pool->rsrc_start) { 1970 vmem_xfree(state->ts_ddrvmem, 1971 rsrc_pool->rsrc_ddr_offset, 1972 rsrc_pool->rsrc_pool_size); 1973 } 1974 TNF_PROBE_0(tavor_rsrc_hw_entries_init_vmemcreate_fail, 1975 TAVOR_TNF_ERROR, ""); 1976 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 1977 return (DDI_FAILURE); 1978 } 1979 rsrc_pool->rsrc_vmp = vmp; 1980 } else { 1981 rsrc_pool->rsrc_vmp = NULL; 1982 } 1983 1984 /* The first HW entries may be reserved by Tavor firmware */ 1985 if (num_prealloc != 0) { 1986 status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type, 1987 num_prealloc, TAVOR_SLEEP, &rsvd_rsrc); 1988 if (status != DDI_SUCCESS) { 1989 /* Unable to preallocate the reserved HW entries */ 1990 if (rsrc_pool->rsrc_vmp != NULL) { 1991 vmem_destroy(rsrc_pool->rsrc_vmp); 1992 } 1993 if (rsrc_pool->rsrc_ddr_offset != 1994 rsrc_pool->rsrc_start) { 1995 vmem_xfree(state->ts_ddrvmem, 1996 rsrc_pool->rsrc_ddr_offset, 1997 rsrc_pool->rsrc_pool_size); 1998 } 1999 TNF_PROBE_0(tavor_rsrc_hw_entries_init_pre_fail, 2000 TAVOR_TNF_ERROR, ""); 2001 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 2002 return (DDI_FAILURE); 2003 } 2004 } 2005 rsrc_pool->rsrc_private = rsvd_rsrc; 2006 2007 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init); 2008 return (DDI_SUCCESS); 2009} 2010 2011 2012/* 2013 * tavor_rsrc_hw_entries_fini() 2014 * Context: Only called from attach() and/or detach() path contexts 2015 */ 2016static void 2017tavor_rsrc_hw_entries_fini(tavor_state_t *state, 2018 tavor_rsrc_hw_entry_info_t *info) 2019{ 2020 tavor_rsrc_pool_info_t *rsrc_pool; 2021 tavor_rsrc_t *rsvd_rsrc; 2022 2023 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_fini); 2024 2025 ASSERT(state != NULL); 2026 ASSERT(info != NULL); 2027 2028 rsrc_pool = info->hwi_rsrcpool; 2029 ASSERT(rsrc_pool != NULL); 2030 2031 /* Free up any "reserved" (i.e. preallocated) HW entries */ 2032 rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private; 2033 if (rsvd_rsrc != NULL) { 2034 tavor_rsrc_free(state, &rsvd_rsrc); 2035 } 2036 2037 /* 2038 * If we've actually setup a vmem arena for the HW entries, then 2039 * destroy it now 2040 */ 2041 if (rsrc_pool->rsrc_vmp != NULL) { 2042 vmem_destroy(rsrc_pool->rsrc_vmp); 2043 } 2044 2045 /* 2046 * Determine if a region was allocated from the tavor_ddr_vmem 2047 * arena (and free it up if necessary) 2048 */ 2049 if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) { 2050 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset, 2051 rsrc_pool->rsrc_pool_size); 2052 } 2053 2054 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_fini); 2055} 2056 2057 2058/* 2059 * tavor_rsrc_sw_handles_init() 2060 * Context: Only called from attach() path context 2061 */ 2062/* ARGSUSED */ 2063static int 2064tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2065{ 2066 tavor_rsrc_pool_info_t *rsrc_pool; 2067 uint64_t num_swhdl, max_swhdl, prealloc_sz; 2068 2069 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_init); 2070 2071 ASSERT(state != NULL); 2072 ASSERT(info != NULL); 2073 2074 rsrc_pool = info->swi_rsrcpool; 2075 ASSERT(rsrc_pool != NULL); 2076 num_swhdl = info->swi_num; 2077 max_swhdl = info->swi_max; 2078 prealloc_sz = info->swi_prealloc_sz; 2079 2080 /* Make sure number of SW handles makes sense */ 2081 if (num_swhdl > max_swhdl) { 2082 TNF_PROBE_2(tavor_rsrc_sw_handles_init_toomany_fail, 2083 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of SW " 2084 "handles exceeds maximum", tnf_uint, maxsw, max_swhdl); 2085 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init); 2086 return (DDI_FAILURE); 2087 } 2088 2089 /* 2090 * Depending on the flags parameter, create a kmem_cache for some 2091 * number of software handle structures. Note: kmem_cache_create() 2092 * will SLEEP until successful. 2093 */ 2094 if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) { 2095 rsrc_pool->rsrc_private = kmem_cache_create( 2096 info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0, 2097 info->swi_constructor, info->swi_destructor, NULL, 2098 rsrc_pool->rsrc_state, NULL, 0); 2099 } 2100 2101 /* Allocate the central list of SW handle pointers */ 2102 if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) { 2103 info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz, 2104 KM_SLEEP); 2105 } 2106 2107 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init); 2108 return (DDI_SUCCESS); 2109} 2110 2111 2112/* 2113 * tavor_rsrc_sw_handles_fini() 2114 * Context: Only called from attach() and/or detach() path contexts 2115 */ 2116/* ARGSUSED */ 2117static void 2118tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2119{ 2120 tavor_rsrc_pool_info_t *rsrc_pool; 2121 uint64_t num_swhdl, prealloc_sz; 2122 2123 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_fini); 2124 2125 ASSERT(state != NULL); 2126 ASSERT(info != NULL); 2127 2128 rsrc_pool = info->swi_rsrcpool; 2129 num_swhdl = info->swi_num; 2130 prealloc_sz = info->swi_prealloc_sz; 2131 2132 /* 2133 * If a "software handle" kmem_cache exists for this resource, then 2134 * destroy it now 2135 */ 2136 if (rsrc_pool != NULL) { 2137 kmem_cache_destroy(rsrc_pool->rsrc_private); 2138 } 2139 2140 /* Free up this central list of SW handle pointers */ 2141 if (info->swi_table_ptr != NULL) { 2142 kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz); 2143 } 2144 2145 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_fini); 2146} 2147 2148 2149/* 2150 * tavor_rsrc_pd_handles_init() 2151 * Context: Only called from attach() path context 2152 */ 2153static int 2154tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2155{ 2156 tavor_rsrc_pool_info_t *rsrc_pool; 2157 vmem_t *vmp; 2158 char vmem_name[TAVOR_RSRC_NAME_MAXLEN]; 2159 int status; 2160 2161 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_init); 2162 2163 ASSERT(state != NULL); 2164 ASSERT(info != NULL); 2165 2166 rsrc_pool = info->swi_rsrcpool; 2167 ASSERT(rsrc_pool != NULL); 2168 2169 /* Initialize the resource pool for software handle table */ 2170 status = tavor_rsrc_sw_handles_init(state, info); 2171 if (status != DDI_SUCCESS) { 2172 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, ""); 2173 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2174 return (DDI_FAILURE); 2175 } 2176 2177 /* Build vmem arena name from Tavor instance */ 2178 TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM); 2179 2180 /* Create new vmem arena for PD numbers */ 2181 vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL, 2182 NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); 2183 if (vmp == NULL) { 2184 /* Unable to create vmem arena */ 2185 info->swi_table_ptr = NULL; 2186 tavor_rsrc_sw_handles_fini(state, info); 2187 TNF_PROBE_0(tavor_rsrc_pd_handles_init_vmem_create_fail, 2188 TAVOR_TNF_ERROR, ""); 2189 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init); 2190 return (DDI_FAILURE); 2191 } 2192 rsrc_pool->rsrc_vmp = vmp; 2193 2194 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init); 2195 return (DDI_SUCCESS); 2196} 2197 2198 2199/* 2200 * tavor_rsrc_pd_handles_fini() 2201 * Context: Only called from attach() and/or detach() path contexts 2202 */ 2203static void 2204tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info) 2205{ 2206 tavor_rsrc_pool_info_t *rsrc_pool; 2207 2208 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_fini); 2209 2210 ASSERT(state != NULL); 2211 ASSERT(info != NULL); 2212 2213 rsrc_pool = info->swi_rsrcpool; 2214 2215 /* Destroy the specially created UAR scratch table vmem arena */ 2216 vmem_destroy(rsrc_pool->rsrc_vmp); 2217 2218 /* Destroy the "tavor_sw_pd_t" kmem_cache */ 2219 tavor_rsrc_sw_handles_fini(state, info); 2220 2221 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_fini); 2222} 2223 2224 2225/* 2226 * tavor_rsrc_mbox_alloc() 2227 * Context: Only called from attach() path context 2228 */ 2229static int 2230tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num, 2231 tavor_rsrc_t *hdl) 2232{ 2233 tavor_rsrc_priv_mbox_t *priv; 2234 void *addr; 2235 caddr_t kaddr; 2236 uint64_t offset; 2237 size_t real_len, temp_len; 2238 int status; 2239 2240 TAVOR_TNF_ENTER(tavor_rsrc_mbox_alloc); 2241 2242 ASSERT(pool_info != NULL); 2243 ASSERT(hdl != NULL); 2244 2245 /* Get the private pointer for the mailboxes */ 2246 priv = pool_info->rsrc_private; 2247 ASSERT(priv != NULL); 2248 2249 /* 2250 * Allocate a DMA handle for the mailbox. This will be used for 2251 * two purposes (potentially). First, it could be used below in 2252 * the call to ddi_dma_mem_alloc() - if the mailbox is to come from 2253 * system memory. Second, it is definitely used later to bind 2254 * the mailbox for DMA access from/by the hardware. 2255 */ 2256 status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr, 2257 DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl); 2258 if (status != DDI_SUCCESS) { 2259 TNF_PROBE_1(tavor_rsrc_mbox_alloc_dmahdl_fail, TAVOR_TNF_ERROR, 2260 "", tnf_uint, status, status); 2261 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2262 return (DDI_FAILURE); 2263 } 2264 2265 /* Is mailbox in DDR memory or system memory? */ 2266 if (pool_info->rsrc_loc == TAVOR_IN_DDR) { 2267 /* Use vmem_alloc() to get DDR address of mbox */ 2268 hdl->tr_len = (num * pool_info->rsrc_quantum); 2269 addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len, 2270 VM_SLEEP); 2271 if (addr == NULL) { 2272 /* No more DDR available for mailbox entries */ 2273 ddi_dma_free_handle(&hdl->tr_dmahdl); 2274 TNF_PROBE_0(tavor_rsrc_mbox_alloc_vma_fail, 2275 TAVOR_TNF_ERROR, ""); 2276 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2277 return (DDI_FAILURE); 2278 } 2279 hdl->tr_acchdl = priv->pmb_acchdl; 2280 2281 /* Calculate kernel virtual address (from the DDR offset) */ 2282 offset = ((uintptr_t)addr - 2283 (uintptr_t)pool_info->rsrc_ddr_offset); 2284 hdl->tr_addr = (void *)(uintptr_t)(offset + 2285 (uintptr_t)pool_info->rsrc_start); 2286 2287 } else { /* TAVOR_IN_SYSMEM */ 2288 2289 /* Use ddi_dma_mem_alloc() to get memory for mailbox */ 2290 temp_len = (num * pool_info->rsrc_quantum); 2291 status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len, 2292 &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP, 2293 NULL, &kaddr, &real_len, &hdl->tr_acchdl); 2294 if (status != DDI_SUCCESS) { 2295 /* No more sys memory available for mailbox entries */ 2296 ddi_dma_free_handle(&hdl->tr_dmahdl); 2297 TNF_PROBE_0(tavor_rsrc_mbox_alloc_dma_memalloc_fail, 2298 TAVOR_TNF_ERROR, ""); 2299 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2300 return (DDI_FAILURE); 2301 } 2302 hdl->tr_addr = (void *)kaddr; 2303 hdl->tr_len = real_len; 2304 } 2305 2306 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc); 2307 return (DDI_SUCCESS); 2308} 2309 2310 2311/* 2312 * tavor_rsrc_mbox_free() 2313 * Context: Can be called from interrupt or base context. 2314 */ 2315static void 2316tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2317{ 2318 void *addr; 2319 uint64_t offset; 2320 2321 TAVOR_TNF_ENTER(tavor_rsrc_mbox_free); 2322 2323 ASSERT(pool_info != NULL); 2324 ASSERT(hdl != NULL); 2325 2326 /* Is mailbox in DDR memory or system memory? */ 2327 if (pool_info->rsrc_loc == TAVOR_IN_DDR) { 2328 2329 /* Calculate the allocated address (the mbox's DDR offset) */ 2330 offset = ((uintptr_t)hdl->tr_addr - 2331 (uintptr_t)pool_info->rsrc_start); 2332 addr = (void *)(uintptr_t)(offset + 2333 (uintptr_t)pool_info->rsrc_ddr_offset); 2334 2335 /* Use vmem_free() to free up DDR memory for mailbox */ 2336 vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len); 2337 2338 } else { /* TAVOR_IN_SYSMEM */ 2339 2340 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */ 2341 ddi_dma_mem_free(&hdl->tr_acchdl); 2342 } 2343 2344 /* Free the DMA handle for the mailbox */ 2345 ddi_dma_free_handle(&hdl->tr_dmahdl); 2346 2347 TAVOR_TNF_EXIT(tavor_rsrc_mbox_free); 2348} 2349 2350 2351/* 2352 * tavor_rsrc_hw_entry_alloc() 2353 * Context: Can be called from interrupt or base context. 2354 */ 2355static int 2356tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num, 2357 uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag, 2358 tavor_rsrc_t *hdl) 2359{ 2360 void *addr; 2361 uint64_t offset; 2362 uint32_t align; 2363 int flag; 2364 2365 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_alloc); 2366 2367 ASSERT(pool_info != NULL); 2368 ASSERT(hdl != NULL); 2369 2370 /* 2371 * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not 2372 * use dma_handle (because they are in Tavor locally attached DDR 2373 * memory) and, generally, don't use the acc_handle (because the 2374 * entries are not directly accessed by software). The exceptions 2375 * to this rule are the UARPG and UDAV entries. 2376 */ 2377 2378 /* 2379 * Use vmem_xalloc() to get a properly aligned pointer (based on 2380 * the number requested) to the HW entry(ies). This handles the 2381 * cases (for special QPCs and for RDB entries) where we need more 2382 * than one and need to ensure that they are properly aligned. 2383 */ 2384 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP; 2385 hdl->tr_len = (num * pool_info->rsrc_quantum); 2386 align = (num_align * pool_info->rsrc_quantum); 2387 addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len, 2388 align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT); 2389 if (addr == NULL) { 2390 /* No more HW entries available */ 2391 TNF_PROBE_0(tavor_rsrc_hw_entry_alloc_vmxa_fail, 2392 TAVOR_TNF_ERROR, ""); 2393 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc); 2394 return (DDI_FAILURE); 2395 } 2396 2397 /* If an access handle was provided, fill it in */ 2398 if (acc_handle != 0) { 2399 hdl->tr_acchdl = acc_handle; 2400 } 2401 2402 /* Calculate vaddr and HW table index (from the DDR offset) */ 2403 offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset); 2404 hdl->tr_addr = (void *)(uintptr_t)(offset + 2405 (uintptr_t)pool_info->rsrc_start); 2406 hdl->tr_indx = (offset >> pool_info->rsrc_shift); 2407 2408 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc); 2409 return (DDI_SUCCESS); 2410} 2411 2412 2413/* 2414 * tavor_rsrc_hw_entry_free() 2415 * Context: Can be called from interrupt or base context. 2416 */ 2417static void 2418tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2419{ 2420 void *addr; 2421 uint64_t offset; 2422 2423 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_free); 2424 2425 ASSERT(pool_info != NULL); 2426 ASSERT(hdl != NULL); 2427 2428 /* Calculate the allocated address (the entry's DDR offset) */ 2429 offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start); 2430 addr = (void *)(uintptr_t)(offset + 2431 (uintptr_t)pool_info->rsrc_ddr_offset); 2432 2433 /* Use vmem_xfree() to free up the HW table entry */ 2434 vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len); 2435 2436 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_free); 2437} 2438 2439 2440/* 2441 * tavor_rsrc_swhdl_alloc() 2442 * Context: Can be called from interrupt or base context. 2443 */ 2444static int 2445tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag, 2446 tavor_rsrc_t *hdl) 2447{ 2448 void *addr; 2449 int flag; 2450 2451 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_alloc); 2452 2453 ASSERT(pool_info != NULL); 2454 ASSERT(hdl != NULL); 2455 2456 /* Allocate the software handle structure */ 2457 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 2458 addr = kmem_cache_alloc(pool_info->rsrc_private, flag); 2459 if (addr == NULL) { 2460 TNF_PROBE_0(tavor_rsrc_swhdl_alloc_kmca_fail, TAVOR_TNF_ERROR, 2461 ""); 2462 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc); 2463 return (DDI_FAILURE); 2464 } 2465 hdl->tr_len = pool_info->rsrc_quantum; 2466 hdl->tr_addr = addr; 2467 2468 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc); 2469 return (DDI_SUCCESS); 2470} 2471 2472 2473/* 2474 * tavor_rsrc_swhdl_free() 2475 * Context: Can be called from interrupt or base context. 2476 */ 2477static void 2478tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2479{ 2480 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_free); 2481 2482 ASSERT(pool_info != NULL); 2483 ASSERT(hdl != NULL); 2484 2485 /* Free the software handle structure */ 2486 kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr); 2487 2488 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_free); 2489} 2490 2491 2492/* 2493 * tavor_rsrc_pdhdl_alloc() 2494 * Context: Can be called from interrupt or base context. 2495 */ 2496static int 2497tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag, 2498 tavor_rsrc_t *hdl) 2499{ 2500 tavor_pdhdl_t addr; 2501 void *tmpaddr; 2502 int flag, status; 2503 2504 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_alloc); 2505 2506 ASSERT(pool_info != NULL); 2507 ASSERT(hdl != NULL); 2508 2509 /* Allocate the software handle */ 2510 status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl); 2511 if (status != DDI_SUCCESS) { 2512 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, ""); 2513 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2514 return (DDI_FAILURE); 2515 } 2516 addr = (tavor_pdhdl_t)hdl->tr_addr; 2517 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr)) 2518 2519 /* Allocate a PD number for the handle */ 2520 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP; 2521 tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag); 2522 if (tmpaddr == NULL) { 2523 /* No more PD number entries available */ 2524 tavor_rsrc_swhdl_free(pool_info, hdl); 2525 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_vma_fail, 2526 TAVOR_TNF_ERROR, ""); 2527 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2528 return (DDI_FAILURE); 2529 } 2530 addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr; 2531 addr->pd_rsrcp = hdl; 2532 hdl->tr_indx = addr->pd_pdnum; 2533 2534 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc); 2535 return (DDI_SUCCESS); 2536} 2537 2538 2539/* 2540 * tavor_rsrc_pdhdl_free() 2541 * Context: Can be called from interrupt or base context. 2542 */ 2543static void 2544tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl) 2545{ 2546 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_free); 2547 2548 ASSERT(pool_info != NULL); 2549 ASSERT(hdl != NULL); 2550 2551 /* Use vmem_free() to free up the PD number */ 2552 vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1); 2553 2554 /* Free the software handle structure */ 2555 tavor_rsrc_swhdl_free(pool_info, hdl); 2556 2557 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_free); 2558} 2559 2560 2561/* 2562 * tavor_rsrc_pdhdl_constructor() 2563 * Context: Can be called from interrupt or base context. 2564 */ 2565/* ARGSUSED */ 2566static int 2567tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags) 2568{ 2569 tavor_pdhdl_t pdhdl; 2570 tavor_state_t *state; 2571 2572 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_constructor); 2573 2574 pdhdl = (tavor_pdhdl_t)pd; 2575 state = (tavor_state_t *)priv; 2576 2577 mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER, 2578 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2579 2580 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_constructor); 2581 return (DDI_SUCCESS); 2582} 2583 2584 2585/* 2586 * tavor_rsrc_pdhdl_destructor() 2587 * Context: Can be called from interrupt or base context. 2588 */ 2589/* ARGSUSED */ 2590static void 2591tavor_rsrc_pdhdl_destructor(void *pd, void *priv) 2592{ 2593 tavor_pdhdl_t pdhdl; 2594 2595 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_destructor); 2596 2597 pdhdl = (tavor_pdhdl_t)pd; 2598 2599 mutex_destroy(&pdhdl->pd_lock); 2600 2601 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_destructor); 2602} 2603 2604 2605/* 2606 * tavor_rsrc_cqhdl_constructor() 2607 * Context: Can be called from interrupt or base context. 2608 */ 2609/* ARGSUSED */ 2610static int 2611tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags) 2612{ 2613 tavor_cqhdl_t cqhdl; 2614 tavor_state_t *state; 2615 2616 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_constructor); 2617 2618 cqhdl = (tavor_cqhdl_t)cq; 2619 state = (tavor_state_t *)priv; 2620 2621 mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER, 2622 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2623 mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER, 2624 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2625 2626 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_constructor); 2627 return (DDI_SUCCESS); 2628} 2629 2630 2631/* 2632 * tavor_rsrc_cqhdl_destructor() 2633 * Context: Can be called from interrupt or base context. 2634 */ 2635/* ARGSUSED */ 2636static void 2637tavor_rsrc_cqhdl_destructor(void *cq, void *priv) 2638{ 2639 tavor_cqhdl_t cqhdl; 2640 2641 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_destructor); 2642 2643 cqhdl = (tavor_cqhdl_t)cq; 2644 2645 mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock); 2646 mutex_destroy(&cqhdl->cq_lock); 2647 2648 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_destructor); 2649} 2650 2651 2652/* 2653 * tavor_rsrc_qphdl_constructor() 2654 * Context: Can be called from interrupt or base context. 2655 */ 2656/* ARGSUSED */ 2657static int 2658tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags) 2659{ 2660 tavor_qphdl_t qphdl; 2661 tavor_state_t *state; 2662 2663 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_constructor); 2664 2665 qphdl = (tavor_qphdl_t)qp; 2666 state = (tavor_state_t *)priv; 2667 2668 mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER, 2669 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2670 2671 TAVOR_TNF_EXIT(tavor_rsrc_qphdl_constructor); 2672 return (DDI_SUCCESS); 2673} 2674 2675 2676/* 2677 * tavor_rsrc_qphdl_destructor() 2678 * Context: Can be called from interrupt or base context. 2679 */ 2680/* ARGSUSED */ 2681static void 2682tavor_rsrc_qphdl_destructor(void *qp, void *priv) 2683{ 2684 tavor_qphdl_t qphdl; 2685 2686 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor); 2687 2688 qphdl = (tavor_qphdl_t)qp; 2689 2690 mutex_destroy(&qphdl->qp_lock); 2691 2692 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor); 2693} 2694 2695 2696/* 2697 * tavor_rsrc_srqhdl_constructor() 2698 * Context: Can be called from interrupt or base context. 2699 */ 2700/* ARGSUSED */ 2701static int 2702tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags) 2703{ 2704 tavor_srqhdl_t srqhdl; 2705 tavor_state_t *state; 2706 2707 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_constructor); 2708 2709 srqhdl = (tavor_srqhdl_t)srq; 2710 state = (tavor_state_t *)priv; 2711 2712 mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER, 2713 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2714 2715 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_constructor); 2716 return (DDI_SUCCESS); 2717} 2718 2719 2720/* 2721 * tavor_rsrc_srqhdl_destructor() 2722 * Context: Can be called from interrupt or base context. 2723 */ 2724/* ARGSUSED */ 2725static void 2726tavor_rsrc_srqhdl_destructor(void *srq, void *priv) 2727{ 2728 tavor_srqhdl_t srqhdl; 2729 2730 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_destructor); 2731 2732 srqhdl = (tavor_srqhdl_t)srq; 2733 2734 mutex_destroy(&srqhdl->srq_lock); 2735 2736 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_destructor); 2737} 2738 2739 2740/* 2741 * tavor_rsrc_refcnt_constructor() 2742 * Context: Can be called from interrupt or base context. 2743 */ 2744/* ARGSUSED */ 2745static int 2746tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags) 2747{ 2748 tavor_sw_refcnt_t *refcnt; 2749 tavor_state_t *state; 2750 2751 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_constructor); 2752 2753 refcnt = (tavor_sw_refcnt_t *)rc; 2754 state = (tavor_state_t *)priv; 2755 2756 mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER, 2757 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2758 2759 TAVOR_TNF_EXIT(tavor_rsrc_refcnt_constructor); 2760 return (DDI_SUCCESS); 2761} 2762 2763 2764/* 2765 * tavor_rsrc_refcnt_destructor() 2766 * Context: Can be called from interrupt or base context. 2767 */ 2768/* ARGSUSED */ 2769static void 2770tavor_rsrc_refcnt_destructor(void *rc, void *priv) 2771{ 2772 tavor_sw_refcnt_t *refcnt; 2773 2774 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor); 2775 2776 refcnt = (tavor_sw_refcnt_t *)rc; 2777 2778 mutex_destroy(&refcnt->swrc_lock); 2779 2780 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor); 2781} 2782 2783 2784/* 2785 * tavor_rsrc_ahhdl_constructor() 2786 * Context: Can be called from interrupt or base context. 2787 */ 2788/* ARGSUSED */ 2789static int 2790tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags) 2791{ 2792 tavor_ahhdl_t ahhdl; 2793 tavor_state_t *state; 2794 2795 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_constructor); 2796 2797 ahhdl = (tavor_ahhdl_t)ah; 2798 state = (tavor_state_t *)priv; 2799 2800 mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER, 2801 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2802 2803 TAVOR_TNF_EXIT(tavor_rsrc_ahhdl_constructor); 2804 return (DDI_SUCCESS); 2805} 2806 2807 2808/* 2809 * tavor_rsrc_ahhdl_destructor() 2810 * Context: Can be called from interrupt or base context. 2811 */ 2812/* ARGSUSED */ 2813static void 2814tavor_rsrc_ahhdl_destructor(void *ah, void *priv) 2815{ 2816 tavor_ahhdl_t ahhdl; 2817 2818 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor); 2819 2820 ahhdl = (tavor_ahhdl_t)ah; 2821 2822 mutex_destroy(&ahhdl->ah_lock); 2823 2824 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor); 2825} 2826 2827 2828/* 2829 * tavor_rsrc_mrhdl_constructor() 2830 * Context: Can be called from interrupt or base context. 2831 */ 2832/* ARGSUSED */ 2833static int 2834tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags) 2835{ 2836 tavor_mrhdl_t mrhdl; 2837 tavor_state_t *state; 2838 2839 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_constructor); 2840 2841 mrhdl = (tavor_mrhdl_t)mr; 2842 state = (tavor_state_t *)priv; 2843 2844 mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER, 2845 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2846 2847 TAVOR_TNF_EXIT(tavor_rsrc_mrhdl_constructor); 2848 return (DDI_SUCCESS); 2849} 2850 2851 2852/* 2853 * tavor_rsrc_mrhdl_destructor() 2854 * Context: Can be called from interrupt or base context. 2855 */ 2856/* ARGSUSED */ 2857static void 2858tavor_rsrc_mrhdl_destructor(void *mr, void *priv) 2859{ 2860 tavor_mrhdl_t mrhdl; 2861 2862 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor); 2863 2864 mrhdl = (tavor_mrhdl_t)mr; 2865 2866 mutex_destroy(&mrhdl->mr_lock); 2867 2868 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor); 2869} 2870 2871 2872/* 2873 * tavor_rsrc_mcg_entry_get_size() 2874 */ 2875static int 2876tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift) 2877{ 2878 uint_t num_qp_per_mcg, max_qp_per_mcg, log2; 2879 2880 TAVOR_TNF_ENTER(tavor_rsrc_mcg_entry_get_size); 2881 2882 /* 2883 * Round the configured number of QP per MCG to next larger 2884 * power-of-2 size and update. 2885 */ 2886 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8; 2887 log2 = highbit(num_qp_per_mcg); 2888 if ((num_qp_per_mcg & (num_qp_per_mcg - 1)) == 0) { 2889 log2 = log2 - 1; 2890 } 2891 state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8; 2892 2893 /* Now make sure number of QP per MCG makes sense */ 2894 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg; 2895 max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg); 2896 if (num_qp_per_mcg > max_qp_per_mcg) { 2897 TNF_PROBE_1(tavor_rsrc_mcg_getsz_toomany_qppermcg_fail, 2898 TAVOR_TNF_ERROR, "", tnf_uint, maxqpmcg, max_qp_per_mcg); 2899 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size); 2900 return (DDI_FAILURE); 2901 } 2902 2903 /* Return the (shift) size of an individual MCG HW entry */ 2904 *mcg_size_shift = log2 + 2; 2905 2906 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size); 2907 return (DDI_SUCCESS); 2908} 2909