1296177Sjhibbits/****************************************************************************** 2296177Sjhibbits 3296177Sjhibbits � 1995-2003, 2004, 2005-2011 Freescale Semiconductor, Inc. 4296177Sjhibbits All rights reserved. 5296177Sjhibbits 6296177Sjhibbits This is proprietary source code of Freescale Semiconductor Inc., 7296177Sjhibbits and its use is subject to the NetComm Device Drivers EULA. 8296177Sjhibbits The copyright notice above does not evidence any actual or intended 9296177Sjhibbits publication of such source code. 10296177Sjhibbits 11296177Sjhibbits ALTERNATIVELY, redistribution and use in source and binary forms, with 12296177Sjhibbits or without modification, are permitted provided that the following 13296177Sjhibbits conditions are met: 14296177Sjhibbits * Redistributions of source code must retain the above copyright 15296177Sjhibbits notice, this list of conditions and the following disclaimer. 16296177Sjhibbits * Redistributions in binary form must reproduce the above copyright 17296177Sjhibbits notice, this list of conditions and the following disclaimer in the 18296177Sjhibbits documentation and/or other materials provided with the distribution. 19296177Sjhibbits * Neither the name of Freescale Semiconductor nor the 20296177Sjhibbits names of its contributors may be used to endorse or promote products 21296177Sjhibbits derived from this software without specific prior written permission. 22296177Sjhibbits 23296177Sjhibbits THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY 24296177Sjhibbits EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 25296177Sjhibbits WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 26296177Sjhibbits DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY 27296177Sjhibbits DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 28296177Sjhibbits (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 29296177Sjhibbits LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 30296177Sjhibbits ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31296177Sjhibbits (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32296177Sjhibbits SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33296177Sjhibbits * 34296177Sjhibbits 35296177Sjhibbits **************************************************************************/ 36296177Sjhibbits/****************************************************************************** 37296177Sjhibbits @File qman_low.c 38296177Sjhibbits 39296177Sjhibbits @Description QM Low-level implementation 40296177Sjhibbits*//***************************************************************************/ 41296177Sjhibbits#include "std_ext.h" 42296177Sjhibbits#include "core_ext.h" 43296177Sjhibbits#include "xx_ext.h" 44296177Sjhibbits#include "error_ext.h" 45296177Sjhibbits 46296177Sjhibbits#include "qman_private.h" 47296177Sjhibbits 48296177Sjhibbits 49296177Sjhibbits/***************************/ 50296177Sjhibbits/* Portal register assists */ 51296177Sjhibbits/***************************/ 52296177Sjhibbits 53296177Sjhibbits/* Cache-inhibited register offsets */ 54296177Sjhibbits#define REG_EQCR_PI_CINH (void *)0x0000 55296177Sjhibbits#define REG_EQCR_CI_CINH (void *)0x0004 56296177Sjhibbits#define REG_EQCR_ITR (void *)0x0008 57296177Sjhibbits#define REG_DQRR_PI_CINH (void *)0x0040 58296177Sjhibbits#define REG_DQRR_CI_CINH (void *)0x0044 59296177Sjhibbits#define REG_DQRR_ITR (void *)0x0048 60296177Sjhibbits#define REG_DQRR_DCAP (void *)0x0050 61296177Sjhibbits#define REG_DQRR_SDQCR (void *)0x0054 62296177Sjhibbits#define REG_DQRR_VDQCR (void *)0x0058 63296177Sjhibbits#define REG_DQRR_PDQCR (void *)0x005c 64296177Sjhibbits#define REG_MR_PI_CINH (void *)0x0080 65296177Sjhibbits#define REG_MR_CI_CINH (void *)0x0084 66296177Sjhibbits#define REG_MR_ITR (void *)0x0088 67296177Sjhibbits#define REG_CFG (void *)0x0100 68296177Sjhibbits#define REG_ISR (void *)0x0e00 69296177Sjhibbits#define REG_IER (void *)0x0e04 70296177Sjhibbits#define REG_ISDR (void *)0x0e08 71296177Sjhibbits#define REG_IIR (void *)0x0e0c 72296177Sjhibbits#define REG_ITPR (void *)0x0e14 73296177Sjhibbits 74296177Sjhibbits/* Cache-enabled register offsets */ 75296177Sjhibbits#define CL_EQCR (void *)0x0000 76296177Sjhibbits#define CL_DQRR (void *)0x1000 77296177Sjhibbits#define CL_MR (void *)0x2000 78296177Sjhibbits#define CL_EQCR_PI_CENA (void *)0x3000 79296177Sjhibbits#define CL_EQCR_CI_CENA (void *)0x3100 80296177Sjhibbits#define CL_DQRR_PI_CENA (void *)0x3200 81296177Sjhibbits#define CL_DQRR_CI_CENA (void *)0x3300 82296177Sjhibbits#define CL_MR_PI_CENA (void *)0x3400 83296177Sjhibbits#define CL_MR_CI_CENA (void *)0x3500 84296177Sjhibbits#define CL_RORI_CENA (void *)0x3600 85296177Sjhibbits#define CL_CR (void *)0x3800 86296177Sjhibbits#define CL_RR0 (void *)0x3900 87296177Sjhibbits#define CL_RR1 (void *)0x3940 88296177Sjhibbits 89296177Sjhibbitsstatic __inline__ void *ptr_ADD(void *a, void *b) 90296177Sjhibbits{ 91296177Sjhibbits return (void *)((uintptr_t)a + (uintptr_t)b); 92296177Sjhibbits} 93296177Sjhibbits 94296177Sjhibbits/* The h/w design requires mappings to be size-aligned so that "add"s can be 95296177Sjhibbits * reduced to "or"s. The primitives below do the same for s/w. */ 96296177Sjhibbits/* Bitwise-OR two pointers */ 97296177Sjhibbitsstatic __inline__ void *ptr_OR(void *a, void *b) 98296177Sjhibbits{ 99296177Sjhibbits return (void *)((uintptr_t)a + (uintptr_t)b); 100296177Sjhibbits} 101296177Sjhibbits 102296177Sjhibbits/* Cache-inhibited register access */ 103296177Sjhibbitsstatic __inline__ uint32_t __qm_in(struct qm_addr *qm, void *offset) 104296177Sjhibbits{ 105296177Sjhibbits uint32_t *tmp = (uint32_t *)ptr_ADD(qm->addr_ci, offset); 106296177Sjhibbits return GET_UINT32(*tmp); 107296177Sjhibbits} 108296177Sjhibbitsstatic __inline__ void __qm_out(struct qm_addr *qm, void *offset, uint32_t val) 109296177Sjhibbits{ 110296177Sjhibbits uint32_t *tmp = (uint32_t *)ptr_ADD(qm->addr_ci, offset); 111296177Sjhibbits WRITE_UINT32(*tmp, val); 112296177Sjhibbits} 113296177Sjhibbits#define qm_in(reg) __qm_in(&portal->addr, REG_##reg) 114296177Sjhibbits#define qm_out(reg, val) __qm_out(&portal->addr, REG_##reg, (uint32_t)val) 115296177Sjhibbits 116296177Sjhibbits/* Convert 'n' cachelines to a pointer value for bitwise OR */ 117296177Sjhibbits#define qm_cl(n) (void *)((n) << 6) 118296177Sjhibbits 119296177Sjhibbits/* Cache-enabled (index) register access */ 120296177Sjhibbitsstatic __inline__ void __qm_cl_touch_ro(struct qm_addr *qm, void *offset) 121296177Sjhibbits{ 122296177Sjhibbits dcbt_ro(ptr_ADD(qm->addr_ce, offset)); 123296177Sjhibbits} 124296177Sjhibbitsstatic __inline__ void __qm_cl_touch_rw(struct qm_addr *qm, void *offset) 125296177Sjhibbits{ 126296177Sjhibbits dcbt_rw(ptr_ADD(qm->addr_ce, offset)); 127296177Sjhibbits} 128296177Sjhibbitsstatic __inline__ uint32_t __qm_cl_in(struct qm_addr *qm, void *offset) 129296177Sjhibbits{ 130296177Sjhibbits uint32_t *tmp = (uint32_t *)ptr_ADD(qm->addr_ce, offset); 131296177Sjhibbits return GET_UINT32(*tmp); 132296177Sjhibbits} 133296177Sjhibbitsstatic __inline__ void __qm_cl_out(struct qm_addr *qm, void *offset, uint32_t val) 134296177Sjhibbits{ 135296177Sjhibbits uint32_t *tmp = (uint32_t *)ptr_ADD(qm->addr_ce, offset); 136296177Sjhibbits WRITE_UINT32(*tmp, val); 137296177Sjhibbits dcbf(tmp); 138296177Sjhibbits} 139296177Sjhibbitsstatic __inline__ void __qm_cl_invalidate(struct qm_addr *qm, void *offset) 140296177Sjhibbits{ 141296177Sjhibbits dcbi(ptr_ADD(qm->addr_ce, offset)); 142296177Sjhibbits} 143296177Sjhibbits#define qm_cl_touch_ro(reg) __qm_cl_touch_ro(&portal->addr, CL_##reg##_CENA) 144296177Sjhibbits#define qm_cl_touch_rw(reg) __qm_cl_touch_rw(&portal->addr, CL_##reg##_CENA) 145296177Sjhibbits#define qm_cl_in(reg) __qm_cl_in(&portal->addr, CL_##reg##_CENA) 146296177Sjhibbits#define qm_cl_out(reg, val) __qm_cl_out(&portal->addr, CL_##reg##_CENA, val) 147296177Sjhibbits#define qm_cl_invalidate(reg) __qm_cl_invalidate(&portal->addr, CL_##reg##_CENA) 148296177Sjhibbits 149296177Sjhibbits/* Cyclic helper for rings. TODO: once we are able to do fine-grain perf 150296177Sjhibbits * analysis, look at using the "extra" bit in the ring index registers to avoid 151296177Sjhibbits * cyclic issues. */ 152296177Sjhibbitsstatic __inline__ uint8_t cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last) 153296177Sjhibbits{ 154296177Sjhibbits /* 'first' is included, 'last' is excluded */ 155296177Sjhibbits if (first <= last) 156296177Sjhibbits return (uint8_t)(last - first); 157296177Sjhibbits return (uint8_t)(ringsize + last - first); 158296177Sjhibbits} 159296177Sjhibbits 160296177Sjhibbitsstatic __inline__ t_Error __qm_portal_bind(struct qm_portal *portal, uint8_t iface) 161296177Sjhibbits{ 162296177Sjhibbits t_Error ret = E_BUSY; 163296177Sjhibbits if (!(portal->config.bound & iface)) { 164296177Sjhibbits portal->config.bound |= iface; 165296177Sjhibbits ret = E_OK; 166296177Sjhibbits } 167296177Sjhibbits return ret; 168296177Sjhibbits} 169296177Sjhibbits 170296177Sjhibbitsstatic __inline__ void __qm_portal_unbind(struct qm_portal *portal, uint8_t iface) 171296177Sjhibbits{ 172296177Sjhibbits#ifdef QM_CHECKING 173296177Sjhibbits ASSERT_COND(portal->config.bound & iface); 174296177Sjhibbits#endif /* QM_CHECKING */ 175296177Sjhibbits portal->config.bound &= ~iface; 176296177Sjhibbits} 177296177Sjhibbits 178296177Sjhibbits/* ---------------- */ 179296177Sjhibbits/* --- EQCR API --- */ 180296177Sjhibbits 181296177Sjhibbits/* It's safer to code in terms of the 'eqcr' object than the 'portal' object, 182296177Sjhibbits * because the latter runs the risk of copy-n-paste errors from other code where 183296177Sjhibbits * we could manipulate some other structure within 'portal'. */ 184296177Sjhibbits/* #define EQCR_API_START() register struct qm_eqcr *eqcr = &portal->eqcr */ 185296177Sjhibbits 186296177Sjhibbits/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */ 187296177Sjhibbits#define EQCR_CARRYCLEAR(p) \ 188296177Sjhibbits (void *)((uintptr_t)(p) & (~(uintptr_t)(QM_EQCR_SIZE << 6))) 189296177Sjhibbits 190296177Sjhibbits/* Bit-wise logic to convert a ring pointer to a ring index */ 191296177Sjhibbitsstatic __inline__ uint8_t EQCR_PTR2IDX(struct qm_eqcr_entry *e) 192296177Sjhibbits{ 193296177Sjhibbits return (uint8_t)(((uint32_t)e >> 6) & (QM_EQCR_SIZE - 1)); 194296177Sjhibbits} 195296177Sjhibbits 196296177Sjhibbits/* Increment the 'cursor' ring pointer, taking 'vbit' into account */ 197296177Sjhibbitsstatic __inline__ void EQCR_INC(struct qm_eqcr *eqcr) 198296177Sjhibbits{ 199296177Sjhibbits /* NB: this is odd-looking, but experiments show that it generates fast 200296177Sjhibbits * code with essentially no branching overheads. We increment to the 201296177Sjhibbits * next EQCR pointer and handle overflow and 'vbit'. */ 202296177Sjhibbits struct qm_eqcr_entry *partial = eqcr->cursor + 1; 203296177Sjhibbits eqcr->cursor = EQCR_CARRYCLEAR(partial); 204296177Sjhibbits if (partial != eqcr->cursor) 205296177Sjhibbits eqcr->vbit ^= QM_EQCR_VERB_VBIT; 206296177Sjhibbits} 207296177Sjhibbits 208296177Sjhibbitsstatic __inline__ t_Error qm_eqcr_init(struct qm_portal *portal, e_QmPortalProduceMode pmode, 209296177Sjhibbits e_QmPortalEqcrConsumeMode cmode) 210296177Sjhibbits{ 211296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 212296177Sjhibbits uint32_t cfg; 213296177Sjhibbits uint8_t pi; 214296177Sjhibbits 215296177Sjhibbits if (__qm_portal_bind(portal, QM_BIND_EQCR)) 216296177Sjhibbits return ERROR_CODE(E_BUSY); 217296177Sjhibbits eqcr->ring = ptr_ADD(portal->addr.addr_ce, CL_EQCR); 218296177Sjhibbits eqcr->ci = (uint8_t)(qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1)); 219296177Sjhibbits qm_cl_invalidate(EQCR_CI); 220296177Sjhibbits pi = (uint8_t)(qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1)); 221296177Sjhibbits eqcr->cursor = eqcr->ring + pi; 222296177Sjhibbits eqcr->vbit = (uint8_t)((qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ? 223296177Sjhibbits QM_EQCR_VERB_VBIT : 0); 224296177Sjhibbits eqcr->available = (uint8_t)(QM_EQCR_SIZE - 1 - 225296177Sjhibbits cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi)); 226296177Sjhibbits eqcr->ithresh = (uint8_t)qm_in(EQCR_ITR); 227296177Sjhibbits 228296177Sjhibbits#ifdef QM_CHECKING 229296177Sjhibbits eqcr->busy = 0; 230296177Sjhibbits eqcr->pmode = pmode; 231296177Sjhibbits eqcr->cmode = cmode; 232296177Sjhibbits#else 233296177Sjhibbits UNUSED(cmode); 234296177Sjhibbits#endif /* QM_CHECKING */ 235296177Sjhibbits cfg = (qm_in(CFG) & 0x00ffffff) | 236296177Sjhibbits ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */ 237296177Sjhibbits qm_out(CFG, cfg); 238296177Sjhibbits return 0; 239296177Sjhibbits} 240296177Sjhibbits 241296177Sjhibbitsstatic __inline__ void qm_eqcr_finish(struct qm_portal *portal) 242296177Sjhibbits{ 243296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 244296177Sjhibbits uint8_t pi = (uint8_t)(qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1)); 245296177Sjhibbits uint8_t ci = (uint8_t)(qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1)); 246296177Sjhibbits 247296177Sjhibbits#ifdef QM_CHECKING 248296177Sjhibbits ASSERT_COND(!eqcr->busy); 249296177Sjhibbits#endif /* QM_CHECKING */ 250296177Sjhibbits if (pi != EQCR_PTR2IDX(eqcr->cursor)) 251296177Sjhibbits REPORT_ERROR(WARNING, E_INVALID_STATE, ("losing uncommitted EQCR entries")); 252296177Sjhibbits if (ci != eqcr->ci) 253296177Sjhibbits REPORT_ERROR(WARNING, E_INVALID_STATE, ("missing existing EQCR completions")); 254296177Sjhibbits if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor)) 255296177Sjhibbits REPORT_ERROR(WARNING, E_INVALID_STATE, ("EQCR destroyed unquiesced")); 256296177Sjhibbits __qm_portal_unbind(portal, QM_BIND_EQCR); 257296177Sjhibbits} 258296177Sjhibbits 259296177Sjhibbitsstatic __inline__ struct qm_eqcr_entry *qm_eqcr_start(struct qm_portal *portal) 260296177Sjhibbits{ 261296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 262296177Sjhibbits#ifdef QM_CHECKING 263296177Sjhibbits ASSERT_COND(!eqcr->busy); 264296177Sjhibbits#endif /* QM_CHECKING */ 265296177Sjhibbits if (!eqcr->available) 266296177Sjhibbits return NULL; 267296177Sjhibbits#ifdef QM_CHECKING 268296177Sjhibbits eqcr->busy = 1; 269296177Sjhibbits#endif /* QM_CHECKING */ 270296177Sjhibbits dcbz_64(eqcr->cursor); 271296177Sjhibbits return eqcr->cursor; 272296177Sjhibbits} 273296177Sjhibbits 274296177Sjhibbitsstatic __inline__ void qm_eqcr_abort(struct qm_portal *portal) 275296177Sjhibbits{ 276296177Sjhibbits#ifdef QM_CHECKING 277296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 278296177Sjhibbits ASSERT_COND(eqcr->busy); 279296177Sjhibbits eqcr->busy = 0; 280296177Sjhibbits#else 281296177Sjhibbits UNUSED(portal); 282296177Sjhibbits#endif /* QM_CHECKING */ 283296177Sjhibbits} 284296177Sjhibbits 285296177Sjhibbitsstatic __inline__ struct qm_eqcr_entry *qm_eqcr_pend_and_next(struct qm_portal *portal, uint8_t myverb) 286296177Sjhibbits{ 287296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 288296177Sjhibbits#ifdef QM_CHECKING 289296177Sjhibbits ASSERT_COND(eqcr->busy); 290296177Sjhibbits ASSERT_COND(eqcr->pmode != e_QmPortalPVB); 291296177Sjhibbits#endif /* QM_CHECKING */ 292296177Sjhibbits if (eqcr->available == 1) 293296177Sjhibbits return NULL; 294296177Sjhibbits eqcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | eqcr->vbit); 295296177Sjhibbits dcbf_64(eqcr->cursor); 296296177Sjhibbits EQCR_INC(eqcr); 297296177Sjhibbits eqcr->available--; 298296177Sjhibbits dcbz_64(eqcr->cursor); 299296177Sjhibbits return eqcr->cursor; 300296177Sjhibbits} 301296177Sjhibbits 302296177Sjhibbits#ifdef QM_CHECKING 303296177Sjhibbits#define EQCR_COMMIT_CHECKS(eqcr) \ 304296177Sjhibbitsdo { \ 305296177Sjhibbits ASSERT_COND(eqcr->busy); \ 306296177Sjhibbits ASSERT_COND(eqcr->cursor->orp == (eqcr->cursor->orp & 0x00ffffff)); \ 307296177Sjhibbits ASSERT_COND(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0x00ffffff)); \ 308296177Sjhibbits} while(0) 309296177Sjhibbits 310296177Sjhibbits#else 311296177Sjhibbits#define EQCR_COMMIT_CHECKS(eqcr) 312296177Sjhibbits#endif /* QM_CHECKING */ 313296177Sjhibbits 314296177Sjhibbits 315296177Sjhibbitsstatic __inline__ void qmPortalEqcrPciCommit(struct qm_portal *portal, uint8_t myverb) 316296177Sjhibbits{ 317296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 318296177Sjhibbits#ifdef QM_CHECKING 319296177Sjhibbits EQCR_COMMIT_CHECKS(eqcr); 320296177Sjhibbits ASSERT_COND(eqcr->pmode == e_QmPortalPCI); 321296177Sjhibbits#endif /* QM_CHECKING */ 322296177Sjhibbits eqcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | eqcr->vbit); 323296177Sjhibbits EQCR_INC(eqcr); 324296177Sjhibbits eqcr->available--; 325296177Sjhibbits dcbf_64(eqcr->cursor); 326296177Sjhibbits hwsync(); 327296177Sjhibbits qm_out(EQCR_PI_CINH, EQCR_PTR2IDX(eqcr->cursor)); 328296177Sjhibbits#ifdef QM_CHECKING 329296177Sjhibbits eqcr->busy = 0; 330296177Sjhibbits#endif /* QM_CHECKING */ 331296177Sjhibbits} 332296177Sjhibbits 333296177Sjhibbitsstatic __inline__ void qmPortalEqcrPcePrefetch(struct qm_portal *portal) 334296177Sjhibbits{ 335296177Sjhibbits#ifdef QM_CHECKING 336296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 337296177Sjhibbits ASSERT_COND(eqcr->pmode == e_QmPortalPCE); 338296177Sjhibbits#endif /* QM_CHECKING */ 339296177Sjhibbits qm_cl_invalidate(EQCR_PI); 340296177Sjhibbits qm_cl_touch_rw(EQCR_PI); 341296177Sjhibbits} 342296177Sjhibbits 343296177Sjhibbitsstatic __inline__ void qmPortalEqcrPceCommit(struct qm_portal *portal, uint8_t myverb) 344296177Sjhibbits{ 345296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 346296177Sjhibbits#ifdef QM_CHECKING 347296177Sjhibbits EQCR_COMMIT_CHECKS(eqcr); 348296177Sjhibbits ASSERT_COND(eqcr->pmode == e_QmPortalPCE); 349296177Sjhibbits#endif /* QM_CHECKING */ 350296177Sjhibbits eqcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | eqcr->vbit); 351296177Sjhibbits EQCR_INC(eqcr); 352296177Sjhibbits eqcr->available--; 353296177Sjhibbits dcbf_64(eqcr->cursor); 354296177Sjhibbits lwsync(); 355296177Sjhibbits qm_cl_out(EQCR_PI, EQCR_PTR2IDX(eqcr->cursor)); 356296177Sjhibbits#ifdef QM_CHECKING 357296177Sjhibbits eqcr->busy = 0; 358296177Sjhibbits#endif /* QM_CHECKING */ 359296177Sjhibbits} 360296177Sjhibbits 361296177Sjhibbitsstatic __inline__ void qmPortalEqcrPvbCommit(struct qm_portal *portal, uint8_t myverb) 362296177Sjhibbits{ 363296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 364296177Sjhibbits struct qm_eqcr_entry *eqcursor; 365296177Sjhibbits#ifdef QM_CHECKING 366296177Sjhibbits EQCR_COMMIT_CHECKS(eqcr); 367296177Sjhibbits ASSERT_COND(eqcr->pmode == e_QmPortalPVB); 368296177Sjhibbits#endif /* QM_CHECKING */ 369296177Sjhibbits lwsync(); 370296177Sjhibbits eqcursor = eqcr->cursor; 371296177Sjhibbits eqcursor->__dont_write_directly__verb = (uint8_t)(myverb | eqcr->vbit); 372296177Sjhibbits dcbf_64(eqcursor); 373296177Sjhibbits EQCR_INC(eqcr); 374296177Sjhibbits eqcr->available--; 375296177Sjhibbits#ifdef QM_CHECKING 376296177Sjhibbits eqcr->busy = 0; 377296177Sjhibbits#endif /* QM_CHECKING */ 378296177Sjhibbits} 379296177Sjhibbits 380296177Sjhibbitsstatic __inline__ uint8_t qmPortalEqcrCciUpdate(struct qm_portal *portal) 381296177Sjhibbits{ 382296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 383296177Sjhibbits uint8_t diff, old_ci = eqcr->ci; 384296177Sjhibbits#ifdef QM_CHECKING 385296177Sjhibbits ASSERT_COND(eqcr->cmode == e_QmPortalEqcrCCI); 386296177Sjhibbits#endif /* QM_CHECKING */ 387296177Sjhibbits eqcr->ci = (uint8_t)(qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1)); 388296177Sjhibbits diff = cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); 389296177Sjhibbits eqcr->available += diff; 390296177Sjhibbits return diff; 391296177Sjhibbits} 392296177Sjhibbits 393296177Sjhibbitsstatic __inline__ void qmPortalEqcrCcePrefetch(struct qm_portal *portal) 394296177Sjhibbits{ 395296177Sjhibbits#ifdef QM_CHECKING 396296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 397296177Sjhibbits ASSERT_COND(eqcr->cmode == e_QmPortalEqcrCCE); 398296177Sjhibbits#endif /* QM_CHECKING */ 399296177Sjhibbits qm_cl_touch_ro(EQCR_CI); 400296177Sjhibbits} 401296177Sjhibbits 402296177Sjhibbitsstatic __inline__ uint8_t qmPortalEqcrCceUpdate(struct qm_portal *portal) 403296177Sjhibbits{ 404296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 405296177Sjhibbits uint8_t diff, old_ci = eqcr->ci; 406296177Sjhibbits#ifdef QM_CHECKING 407296177Sjhibbits ASSERT_COND(eqcr->cmode == e_QmPortalEqcrCCE); 408296177Sjhibbits#endif /* QM_CHECKING */ 409296177Sjhibbits eqcr->ci = (uint8_t)(qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1)); 410296177Sjhibbits qm_cl_invalidate(EQCR_CI); 411296177Sjhibbits diff = cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); 412296177Sjhibbits eqcr->available += diff; 413296177Sjhibbits return diff; 414296177Sjhibbits} 415296177Sjhibbits 416296177Sjhibbitsstatic __inline__ uint8_t qm_eqcr_get_ithresh(struct qm_portal *portal) 417296177Sjhibbits{ 418296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 419296177Sjhibbits return eqcr->ithresh; 420296177Sjhibbits} 421296177Sjhibbits 422296177Sjhibbitsstatic __inline__ void qm_eqcr_set_ithresh(struct qm_portal *portal, uint8_t ithresh) 423296177Sjhibbits{ 424296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 425296177Sjhibbits eqcr->ithresh = ithresh; 426296177Sjhibbits qm_out(EQCR_ITR, ithresh); 427296177Sjhibbits} 428296177Sjhibbits 429296177Sjhibbitsstatic __inline__ uint8_t qm_eqcr_get_avail(struct qm_portal *portal) 430296177Sjhibbits{ 431296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 432296177Sjhibbits return eqcr->available; 433296177Sjhibbits} 434296177Sjhibbits 435296177Sjhibbitsstatic __inline__ uint8_t qm_eqcr_get_fill(struct qm_portal *portal) 436296177Sjhibbits{ 437296177Sjhibbits register struct qm_eqcr *eqcr = &portal->eqcr; 438296177Sjhibbits return (uint8_t)(QM_EQCR_SIZE - 1 - eqcr->available); 439296177Sjhibbits} 440296177Sjhibbits 441296177Sjhibbits 442296177Sjhibbits 443296177Sjhibbits/* ---------------- */ 444296177Sjhibbits/* --- DQRR API --- */ 445296177Sjhibbits 446296177Sjhibbits/* TODO: many possible improvements; 447296177Sjhibbits * - look at changing the API to use pointer rather than index parameters now 448296177Sjhibbits * that 'cursor' is a pointer, 449296177Sjhibbits * - consider moving other parameters to pointer if it could help (ci) 450296177Sjhibbits */ 451296177Sjhibbits 452296177Sjhibbits/* It's safer to code in terms of the 'dqrr' object than the 'portal' object, 453296177Sjhibbits * because the latter runs the risk of copy-n-paste errors from other code where 454296177Sjhibbits * we could manipulate some other structure within 'portal'. */ 455296177Sjhibbits/* #define DQRR_API_START() register struct qm_dqrr *dqrr = &portal->dqrr */ 456296177Sjhibbits 457296177Sjhibbits#define DQRR_CARRYCLEAR(p) \ 458296177Sjhibbits (void *)((uintptr_t)(p) & (~(uintptr_t)(QM_DQRR_SIZE << 6))) 459296177Sjhibbits 460296177Sjhibbitsstatic __inline__ uint8_t DQRR_PTR2IDX(struct qm_dqrr_entry *e) 461296177Sjhibbits{ 462296177Sjhibbits return (uint8_t)(((uint32_t)e >> 6) & (QM_DQRR_SIZE - 1)); 463296177Sjhibbits} 464296177Sjhibbits 465296177Sjhibbitsstatic __inline__ struct qm_dqrr_entry *DQRR_INC(struct qm_dqrr_entry *e) 466296177Sjhibbits{ 467296177Sjhibbits return DQRR_CARRYCLEAR(e + 1); 468296177Sjhibbits} 469296177Sjhibbits 470296177Sjhibbitsstatic __inline__ void qm_dqrr_set_maxfill(struct qm_portal *portal, uint8_t mf) 471296177Sjhibbits{ 472296177Sjhibbits qm_out(CFG, (qm_in(CFG) & 0xff0fffff) | 473296177Sjhibbits ((mf & (QM_DQRR_SIZE - 1)) << 20)); 474296177Sjhibbits} 475296177Sjhibbits 476296177Sjhibbitsstatic __inline__ t_Error qm_dqrr_init(struct qm_portal *portal, e_QmPortalDequeueMode dmode, 477296177Sjhibbits e_QmPortalProduceMode pmode, e_QmPortalDqrrConsumeMode cmode, 478296177Sjhibbits uint8_t max_fill, int stash_ring, int stash_data) 479296177Sjhibbits{ 480296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 481296177Sjhibbits const struct qm_portal_config *config = &portal->config; 482296177Sjhibbits uint32_t cfg; 483296177Sjhibbits 484296177Sjhibbits if (__qm_portal_bind(portal, QM_BIND_DQRR)) 485296177Sjhibbits return ERROR_CODE(E_BUSY); 486296177Sjhibbits if ((stash_ring || stash_data) && (config->cpu == -1)) 487296177Sjhibbits return ERROR_CODE(E_INVALID_STATE); 488296177Sjhibbits /* Make sure the DQRR will be idle when we enable */ 489296177Sjhibbits qm_out(DQRR_SDQCR, 0); 490296177Sjhibbits qm_out(DQRR_VDQCR, 0); 491296177Sjhibbits qm_out(DQRR_PDQCR, 0); 492296177Sjhibbits dqrr->ring = ptr_ADD(portal->addr.addr_ce, CL_DQRR); 493296177Sjhibbits dqrr->pi = (uint8_t)(qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1)); 494296177Sjhibbits dqrr->ci = (uint8_t)(qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1)); 495296177Sjhibbits dqrr->cursor = dqrr->ring + dqrr->ci; 496296177Sjhibbits dqrr->fill = cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); 497296177Sjhibbits dqrr->vbit = (uint8_t)((qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ? 498296177Sjhibbits QM_DQRR_VERB_VBIT : 0); 499296177Sjhibbits dqrr->ithresh = (uint8_t)qm_in(DQRR_ITR); 500296177Sjhibbits 501296177Sjhibbits#ifdef QM_CHECKING 502296177Sjhibbits dqrr->dmode = dmode; 503296177Sjhibbits dqrr->pmode = pmode; 504296177Sjhibbits dqrr->cmode = cmode; 505296177Sjhibbits dqrr->flags = 0; 506296177Sjhibbits if (stash_ring) 507296177Sjhibbits dqrr->flags |= QM_DQRR_FLAG_RE; 508296177Sjhibbits if (stash_data) 509296177Sjhibbits dqrr->flags |= QM_DQRR_FLAG_SE; 510296177Sjhibbits#else 511296177Sjhibbits UNUSED(pmode); 512296177Sjhibbits#endif /* QM_CHECKING */ 513296177Sjhibbits 514296177Sjhibbits cfg = (qm_in(CFG) & 0xff000f00) | 515296177Sjhibbits ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */ 516296177Sjhibbits ((dmode & 1) << 18) | /* DP */ 517296177Sjhibbits ((cmode & 3) << 16) | /* DCM */ 518296177Sjhibbits (stash_ring ? 0x80 : 0) | /* RE */ 519296177Sjhibbits (0 ? 0x40 : 0) | /* Ignore RP */ 520296177Sjhibbits (stash_data ? 0x20 : 0) | /* SE */ 521296177Sjhibbits (0 ? 0x10 : 0); /* Ignore SP */ 522296177Sjhibbits qm_out(CFG, cfg); 523296177Sjhibbits return E_OK; 524296177Sjhibbits} 525296177Sjhibbits 526296177Sjhibbits 527296177Sjhibbitsstatic __inline__ void qm_dqrr_finish(struct qm_portal *portal) 528296177Sjhibbits{ 529296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 530296177Sjhibbits if (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor)) 531296177Sjhibbits REPORT_ERROR(WARNING, E_INVALID_STATE, ("Ignoring completed DQRR entries")); 532296177Sjhibbits __qm_portal_unbind(portal, QM_BIND_DQRR); 533296177Sjhibbits} 534296177Sjhibbits 535296177Sjhibbitsstatic __inline__ struct qm_dqrr_entry *qm_dqrr_current(struct qm_portal *portal) 536296177Sjhibbits{ 537296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 538296177Sjhibbits if (!dqrr->fill) 539296177Sjhibbits return NULL; 540296177Sjhibbits return dqrr->cursor; 541296177Sjhibbits} 542296177Sjhibbits 543296177Sjhibbitsstatic __inline__ uint8_t qm_dqrr_cursor(struct qm_portal *portal) 544296177Sjhibbits{ 545296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 546296177Sjhibbits return DQRR_PTR2IDX(dqrr->cursor); 547296177Sjhibbits} 548296177Sjhibbits 549296177Sjhibbitsstatic __inline__ uint8_t qm_dqrr_next(struct qm_portal *portal) 550296177Sjhibbits{ 551296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 552296177Sjhibbits#ifdef QM_CHECKING 553296177Sjhibbits ASSERT_COND(dqrr->fill); 554296177Sjhibbits#endif 555296177Sjhibbits dqrr->cursor = DQRR_INC(dqrr->cursor); 556296177Sjhibbits return --dqrr->fill; 557296177Sjhibbits} 558296177Sjhibbits 559296177Sjhibbitsstatic __inline__ uint8_t qmPortalDqrrPciUpdate(struct qm_portal *portal) 560296177Sjhibbits{ 561296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 562296177Sjhibbits uint8_t diff, old_pi = dqrr->pi; 563296177Sjhibbits#ifdef QM_CHECKING 564296177Sjhibbits ASSERT_COND(dqrr->pmode == e_QmPortalPCI); 565296177Sjhibbits#endif /* QM_CHECKING */ 566296177Sjhibbits dqrr->pi = (uint8_t)(qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1)); 567296177Sjhibbits diff = cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi); 568296177Sjhibbits dqrr->fill += diff; 569296177Sjhibbits return diff; 570296177Sjhibbits} 571296177Sjhibbits 572296177Sjhibbitsstatic __inline__ void qmPortalDqrrPcePrefetch(struct qm_portal *portal) 573296177Sjhibbits{ 574296177Sjhibbits#ifdef QM_CHECKING 575296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 576296177Sjhibbits ASSERT_COND(dqrr->pmode == e_QmPortalPCE); 577296177Sjhibbits#endif /* QM_CHECKING */ 578296177Sjhibbits qm_cl_invalidate(DQRR_PI); 579296177Sjhibbits qm_cl_touch_ro(DQRR_PI); 580296177Sjhibbits} 581296177Sjhibbits 582296177Sjhibbitsstatic __inline__ uint8_t qmPortalDqrrPceUpdate(struct qm_portal *portal) 583296177Sjhibbits{ 584296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 585296177Sjhibbits uint8_t diff, old_pi = dqrr->pi; 586296177Sjhibbits#ifdef QM_CHECKING 587296177Sjhibbits ASSERT_COND(dqrr->pmode == e_QmPortalPCE); 588296177Sjhibbits#endif /* QM_CHECKING */ 589296177Sjhibbits dqrr->pi = (uint8_t)(qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1)); 590296177Sjhibbits diff = cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi); 591296177Sjhibbits dqrr->fill += diff; 592296177Sjhibbits return diff; 593296177Sjhibbits} 594296177Sjhibbits 595296177Sjhibbitsstatic __inline__ void qmPortalDqrrPvbPrefetch(struct qm_portal *portal) 596296177Sjhibbits{ 597296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 598296177Sjhibbits#ifdef QM_CHECKING 599296177Sjhibbits ASSERT_COND(dqrr->pmode == e_QmPortalPVB); 600296177Sjhibbits /* If ring entries get stashed, don't invalidate/prefetch */ 601296177Sjhibbits if (!(dqrr->flags & QM_DQRR_FLAG_RE)) 602296177Sjhibbits#endif /*QM_CHECKING */ 603296177Sjhibbits dcbit_ro(ptr_ADD(dqrr->ring, qm_cl(dqrr->pi))); 604296177Sjhibbits} 605296177Sjhibbits 606296177Sjhibbitsstatic __inline__ uint8_t qmPortalDqrrPvbUpdate(struct qm_portal *portal) 607296177Sjhibbits{ 608296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 609296177Sjhibbits struct qm_dqrr_entry *res = ptr_ADD(dqrr->ring, qm_cl(dqrr->pi)); 610296177Sjhibbits#ifdef QM_CHECKING 611296177Sjhibbits ASSERT_COND(dqrr->pmode == e_QmPortalPVB); 612296177Sjhibbits#endif /* QM_CHECKING */ 613296177Sjhibbits if ((res->verb & QM_DQRR_VERB_VBIT) == dqrr->vbit) { 614296177Sjhibbits dqrr->pi = (uint8_t)((dqrr->pi + 1) & (QM_DQRR_SIZE - 1)); 615296177Sjhibbits if (!dqrr->pi) 616296177Sjhibbits dqrr->vbit ^= QM_DQRR_VERB_VBIT; 617296177Sjhibbits dqrr->fill++; 618296177Sjhibbits return 1; 619296177Sjhibbits } 620296177Sjhibbits return 0; 621296177Sjhibbits} 622296177Sjhibbits 623296177Sjhibbitsstatic __inline__ void qmPortalDqrrCciConsume(struct qm_portal *portal, uint8_t num) 624296177Sjhibbits{ 625296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 626296177Sjhibbits#ifdef QM_CHECKING 627296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrCCI); 628296177Sjhibbits#endif /* QM_CHECKING */ 629296177Sjhibbits dqrr->ci = (uint8_t)((dqrr->ci + num) & (QM_DQRR_SIZE - 1)); 630296177Sjhibbits qm_out(DQRR_CI_CINH, dqrr->ci); 631296177Sjhibbits} 632296177Sjhibbits 633296177Sjhibbitsstatic __inline__ void qmPortalDqrrCciConsumeToCurrent(struct qm_portal *portal) 634296177Sjhibbits{ 635296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 636296177Sjhibbits#ifdef QM_CHECKING 637296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrCCI); 638296177Sjhibbits#endif /* QM_CHECKING */ 639296177Sjhibbits dqrr->ci = DQRR_PTR2IDX(dqrr->cursor); 640296177Sjhibbits qm_out(DQRR_CI_CINH, dqrr->ci); 641296177Sjhibbits} 642296177Sjhibbits 643296177Sjhibbitsstatic __inline__ void qmPortalDqrrCcePrefetch(struct qm_portal *portal) 644296177Sjhibbits{ 645296177Sjhibbits#ifdef QM_CHECKING 646296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 647296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrCCE); 648296177Sjhibbits#endif /* QM_CHECKING */ 649296177Sjhibbits qm_cl_invalidate(DQRR_CI); 650296177Sjhibbits qm_cl_touch_rw(DQRR_CI); 651296177Sjhibbits} 652296177Sjhibbits 653296177Sjhibbitsstatic __inline__ void qmPortalDqrrCceConsume(struct qm_portal *portal, uint8_t num) 654296177Sjhibbits{ 655296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 656296177Sjhibbits#ifdef QM_CHECKING 657296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrCCE); 658296177Sjhibbits#endif /* QM_CHECKING */ 659296177Sjhibbits dqrr->ci = (uint8_t)((dqrr->ci + num) & (QM_DQRR_SIZE - 1)); 660296177Sjhibbits qm_cl_out(DQRR_CI, dqrr->ci); 661296177Sjhibbits} 662296177Sjhibbits 663296177Sjhibbitsstatic __inline__ void qmPortalDqrrCceConsume_to_current(struct qm_portal *portal) 664296177Sjhibbits{ 665296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 666296177Sjhibbits#ifdef QM_CHECKING 667296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrCCE); 668296177Sjhibbits#endif /* QM_CHECKING */ 669296177Sjhibbits dqrr->ci = DQRR_PTR2IDX(dqrr->cursor); 670296177Sjhibbits qm_cl_out(DQRR_CI, dqrr->ci); 671296177Sjhibbits} 672296177Sjhibbits 673296177Sjhibbitsstatic __inline__ void qmPortalDqrrDcaConsume1(struct qm_portal *portal, uint8_t idx, bool park) 674296177Sjhibbits{ 675296177Sjhibbits#ifdef QM_CHECKING 676296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 677296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA); 678296177Sjhibbits#endif /* QM_CHECKING */ 679296177Sjhibbits ASSERT_COND(idx < QM_DQRR_SIZE); 680296177Sjhibbits qm_out(DQRR_DCAP, (0 << 8) | /* S */ 681296177Sjhibbits ((uint32_t)(park ? 1 : 0) << 6) | /* PK */ 682296177Sjhibbits idx); /* DCAP_CI */ 683296177Sjhibbits} 684296177Sjhibbits 685296177Sjhibbitsstatic __inline__ void qmPortalDqrrDcaConsume1ptr(struct qm_portal *portal, 686296177Sjhibbits struct qm_dqrr_entry *dq, 687296177Sjhibbits bool park) 688296177Sjhibbits{ 689296177Sjhibbits uint8_t idx = DQRR_PTR2IDX(dq); 690296177Sjhibbits#ifdef QM_CHECKING 691296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 692296177Sjhibbits 693296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA); 694296177Sjhibbits ASSERT_COND((dqrr->ring + idx) == dq); 695296177Sjhibbits ASSERT_COND(idx < QM_DQRR_SIZE); 696296177Sjhibbits#endif /* QM_CHECKING */ 697296177Sjhibbits qm_out(DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */ 698296177Sjhibbits ((uint32_t)(park ? 1 : 0) << 6) | /* DQRR_DCAP::PK */ 699296177Sjhibbits idx); /* DQRR_DCAP::DCAP_CI */ 700296177Sjhibbits} 701296177Sjhibbits 702296177Sjhibbitsstatic __inline__ void qmPortalDqrrDcaConsumeN(struct qm_portal *portal, uint16_t bitmask) 703296177Sjhibbits{ 704296177Sjhibbits#ifdef QM_CHECKING 705296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 706296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA); 707296177Sjhibbits#endif /* QM_CHECKING */ 708296177Sjhibbits qm_out(DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */ 709296177Sjhibbits ((uint32_t)bitmask << 16)); /* DQRR_DCAP::DCAP_CI */ 710296177Sjhibbits} 711296177Sjhibbits 712296177Sjhibbitsstatic __inline__ uint8_t qmPortalDqrrDcaCci(struct qm_portal *portal) 713296177Sjhibbits{ 714296177Sjhibbits#ifdef QM_CHECKING 715296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 716296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA); 717296177Sjhibbits#endif /* QM_CHECKING */ 718296177Sjhibbits return (uint8_t)(qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1)); 719296177Sjhibbits} 720296177Sjhibbits 721296177Sjhibbitsstatic __inline__ void qmPortalDqrrDcaCcePrefetch(struct qm_portal *portal) 722296177Sjhibbits{ 723296177Sjhibbits#ifdef QM_CHECKING 724296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 725296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA); 726296177Sjhibbits#endif /* QM_CHECKING */ 727296177Sjhibbits qm_cl_invalidate(DQRR_CI); 728296177Sjhibbits qm_cl_touch_ro(DQRR_CI); 729296177Sjhibbits} 730296177Sjhibbits 731296177Sjhibbitsstatic __inline__ uint8_t qmPortalDqrrDcaCce(struct qm_portal *portal) 732296177Sjhibbits{ 733296177Sjhibbits#ifdef QM_CHECKING 734296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 735296177Sjhibbits ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA); 736296177Sjhibbits#endif /* QM_CHECKING */ 737296177Sjhibbits return (uint8_t)(qm_cl_in(DQRR_CI) & (QM_DQRR_SIZE - 1)); 738296177Sjhibbits} 739296177Sjhibbits 740296177Sjhibbitsstatic __inline__ uint8_t qm_dqrr_get_ci(struct qm_portal *portal) 741296177Sjhibbits{ 742296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 743296177Sjhibbits#ifdef QM_CHECKING 744296177Sjhibbits ASSERT_COND(dqrr->cmode != e_QmPortalDqrrDCA); 745296177Sjhibbits#endif /* QM_CHECKING */ 746296177Sjhibbits 747296177Sjhibbits return dqrr->ci; 748296177Sjhibbits} 749296177Sjhibbits 750296177Sjhibbitsstatic __inline__ void qm_dqrr_park(struct qm_portal *portal, uint8_t idx) 751296177Sjhibbits{ 752296177Sjhibbits#ifdef QM_CHECKING 753296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 754296177Sjhibbits ASSERT_COND(dqrr->cmode != e_QmPortalDqrrDCA); 755296177Sjhibbits#endif /* QM_CHECKING */ 756296177Sjhibbits 757296177Sjhibbits qm_out(DQRR_DCAP, (0 << 8) | /* S */ 758296177Sjhibbits (uint32_t)(1 << 6) | /* PK */ 759296177Sjhibbits (idx & (QM_DQRR_SIZE - 1))); /* DCAP_CI */ 760296177Sjhibbits} 761296177Sjhibbits 762296177Sjhibbitsstatic __inline__ void qm_dqrr_park_ci(struct qm_portal *portal) 763296177Sjhibbits{ 764296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 765296177Sjhibbits#ifdef QM_CHECKING 766296177Sjhibbits ASSERT_COND(dqrr->cmode != e_QmPortalDqrrDCA); 767296177Sjhibbits#endif /* QM_CHECKING */ 768296177Sjhibbits qm_out(DQRR_DCAP, (0 << 8) | /* S */ 769296177Sjhibbits (uint32_t)(1 << 6) | /* PK */ 770296177Sjhibbits (dqrr->ci & (QM_DQRR_SIZE - 1)));/* DCAP_CI */ 771296177Sjhibbits} 772296177Sjhibbits 773296177Sjhibbitsstatic __inline__ void qm_dqrr_sdqcr_set(struct qm_portal *portal, uint32_t sdqcr) 774296177Sjhibbits{ 775296177Sjhibbits qm_out(DQRR_SDQCR, sdqcr); 776296177Sjhibbits} 777296177Sjhibbits 778296177Sjhibbitsstatic __inline__ uint32_t qm_dqrr_sdqcr_get(struct qm_portal *portal) 779296177Sjhibbits{ 780296177Sjhibbits return qm_in(DQRR_SDQCR); 781296177Sjhibbits} 782296177Sjhibbits 783296177Sjhibbitsstatic __inline__ void qm_dqrr_vdqcr_set(struct qm_portal *portal, uint32_t vdqcr) 784296177Sjhibbits{ 785296177Sjhibbits qm_out(DQRR_VDQCR, vdqcr); 786296177Sjhibbits} 787296177Sjhibbits 788296177Sjhibbitsstatic __inline__ uint32_t qm_dqrr_vdqcr_get(struct qm_portal *portal) 789296177Sjhibbits{ 790296177Sjhibbits return qm_in(DQRR_VDQCR); 791296177Sjhibbits} 792296177Sjhibbits 793296177Sjhibbitsstatic __inline__ void qm_dqrr_pdqcr_set(struct qm_portal *portal, uint32_t pdqcr) 794296177Sjhibbits{ 795296177Sjhibbits qm_out(DQRR_PDQCR, pdqcr); 796296177Sjhibbits} 797296177Sjhibbits 798296177Sjhibbitsstatic __inline__ uint32_t qm_dqrr_pdqcr_get(struct qm_portal *portal) 799296177Sjhibbits{ 800296177Sjhibbits return qm_in(DQRR_PDQCR); 801296177Sjhibbits} 802296177Sjhibbits 803296177Sjhibbitsstatic __inline__ uint8_t qm_dqrr_get_ithresh(struct qm_portal *portal) 804296177Sjhibbits{ 805296177Sjhibbits register struct qm_dqrr *dqrr = &portal->dqrr; 806296177Sjhibbits return dqrr->ithresh; 807296177Sjhibbits} 808296177Sjhibbits 809296177Sjhibbitsstatic __inline__ void qm_dqrr_set_ithresh(struct qm_portal *portal, uint8_t ithresh) 810296177Sjhibbits{ 811296177Sjhibbits qm_out(DQRR_ITR, ithresh); 812296177Sjhibbits} 813296177Sjhibbits 814296177Sjhibbitsstatic __inline__ uint8_t qm_dqrr_get_maxfill(struct qm_portal *portal) 815296177Sjhibbits{ 816296177Sjhibbits return (uint8_t)((qm_in(CFG) & 0x00f00000) >> 20); 817296177Sjhibbits} 818296177Sjhibbits 819296177Sjhibbits/* -------------- */ 820296177Sjhibbits/* --- MR API --- */ 821296177Sjhibbits 822296177Sjhibbits/* It's safer to code in terms of the 'mr' object than the 'portal' object, 823296177Sjhibbits * because the latter runs the risk of copy-n-paste errors from other code where 824296177Sjhibbits * we could manipulate some other structure within 'portal'. */ 825296177Sjhibbits/* #define MR_API_START() register struct qm_mr *mr = &portal->mr */ 826296177Sjhibbits 827296177Sjhibbits#define MR_CARRYCLEAR(p) \ 828296177Sjhibbits (void *)((uintptr_t)(p) & (~(uintptr_t)(QM_MR_SIZE << 6))) 829296177Sjhibbits 830296177Sjhibbitsstatic __inline__ uint8_t MR_PTR2IDX(struct qm_mr_entry *e) 831296177Sjhibbits{ 832296177Sjhibbits return (uint8_t)(((uint32_t)e >> 6) & (QM_MR_SIZE - 1)); 833296177Sjhibbits} 834296177Sjhibbits 835296177Sjhibbitsstatic __inline__ struct qm_mr_entry *MR_INC(struct qm_mr_entry *e) 836296177Sjhibbits{ 837296177Sjhibbits return MR_CARRYCLEAR(e + 1); 838296177Sjhibbits} 839296177Sjhibbits 840296177Sjhibbitsstatic __inline__ t_Error qm_mr_init(struct qm_portal *portal, e_QmPortalProduceMode pmode, 841296177Sjhibbits e_QmPortalMrConsumeMode cmode) 842296177Sjhibbits{ 843296177Sjhibbits register struct qm_mr *mr = &portal->mr; 844296177Sjhibbits uint32_t cfg; 845296177Sjhibbits 846296177Sjhibbits if (__qm_portal_bind(portal, QM_BIND_MR)) 847296177Sjhibbits return ERROR_CODE(E_BUSY); 848296177Sjhibbits mr->ring = ptr_ADD(portal->addr.addr_ce, CL_MR); 849296177Sjhibbits mr->pi = (uint8_t)(qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1)); 850296177Sjhibbits mr->ci = (uint8_t)(qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1)); 851296177Sjhibbits mr->cursor = mr->ring + mr->ci; 852296177Sjhibbits mr->fill = cyc_diff(QM_MR_SIZE, mr->ci, mr->pi); 853296177Sjhibbits mr->vbit = (uint8_t)((qm_in(MR_PI_CINH) & QM_MR_SIZE) ?QM_MR_VERB_VBIT : 0); 854296177Sjhibbits mr->ithresh = (uint8_t)qm_in(MR_ITR); 855296177Sjhibbits 856296177Sjhibbits#ifdef QM_CHECKING 857296177Sjhibbits mr->pmode = pmode; 858296177Sjhibbits mr->cmode = cmode; 859296177Sjhibbits#else 860296177Sjhibbits UNUSED(pmode); 861296177Sjhibbits#endif /* QM_CHECKING */ 862296177Sjhibbits cfg = (qm_in(CFG) & 0xfffff0ff) | 863296177Sjhibbits ((cmode & 1) << 8); /* QCSP_CFG:MM */ 864296177Sjhibbits qm_out(CFG, cfg); 865296177Sjhibbits return E_OK; 866296177Sjhibbits} 867296177Sjhibbits 868296177Sjhibbits 869296177Sjhibbitsstatic __inline__ void qm_mr_finish(struct qm_portal *portal) 870296177Sjhibbits{ 871296177Sjhibbits register struct qm_mr *mr = &portal->mr; 872296177Sjhibbits if (mr->ci != MR_PTR2IDX(mr->cursor)) 873296177Sjhibbits REPORT_ERROR(WARNING, E_INVALID_STATE, ("Ignoring completed MR entries")); 874296177Sjhibbits __qm_portal_unbind(portal, QM_BIND_MR); 875296177Sjhibbits} 876296177Sjhibbits 877296177Sjhibbitsstatic __inline__ void qm_mr_current_prefetch(struct qm_portal *portal) 878296177Sjhibbits{ 879296177Sjhibbits register struct qm_mr *mr = &portal->mr; 880296177Sjhibbits dcbt_ro(mr->cursor); 881296177Sjhibbits} 882296177Sjhibbits 883296177Sjhibbitsstatic __inline__ struct qm_mr_entry *qm_mr_current(struct qm_portal *portal) 884296177Sjhibbits{ 885296177Sjhibbits register struct qm_mr *mr = &portal->mr; 886296177Sjhibbits if (!mr->fill) 887296177Sjhibbits return NULL; 888296177Sjhibbits return mr->cursor; 889296177Sjhibbits} 890296177Sjhibbits 891296177Sjhibbitsstatic __inline__ uint8_t qm_mr_cursor(struct qm_portal *portal) 892296177Sjhibbits{ 893296177Sjhibbits register struct qm_mr *mr = &portal->mr; 894296177Sjhibbits return MR_PTR2IDX(mr->cursor); 895296177Sjhibbits} 896296177Sjhibbits 897296177Sjhibbitsstatic __inline__ uint8_t qm_mr_next(struct qm_portal *portal) 898296177Sjhibbits{ 899296177Sjhibbits register struct qm_mr *mr = &portal->mr; 900296177Sjhibbits#ifdef QM_CHECKING 901296177Sjhibbits ASSERT_COND(mr->fill); 902296177Sjhibbits#endif /* QM_CHECKING */ 903296177Sjhibbits mr->cursor = MR_INC(mr->cursor); 904296177Sjhibbits return --mr->fill; 905296177Sjhibbits} 906296177Sjhibbits 907296177Sjhibbitsstatic __inline__ uint8_t qmPortalMrPciUpdate(struct qm_portal *portal) 908296177Sjhibbits{ 909296177Sjhibbits register struct qm_mr *mr = &portal->mr; 910296177Sjhibbits uint8_t diff, old_pi = mr->pi; 911296177Sjhibbits#ifdef QM_CHECKING 912296177Sjhibbits ASSERT_COND(mr->pmode == e_QmPortalPCI); 913296177Sjhibbits#endif /* QM_CHECKING */ 914296177Sjhibbits mr->pi = (uint8_t)qm_in(MR_PI_CINH); 915296177Sjhibbits diff = cyc_diff(QM_MR_SIZE, old_pi, mr->pi); 916296177Sjhibbits mr->fill += diff; 917296177Sjhibbits return diff; 918296177Sjhibbits} 919296177Sjhibbits 920296177Sjhibbitsstatic __inline__ void qmPortalMrPcePrefetch(struct qm_portal *portal) 921296177Sjhibbits{ 922296177Sjhibbits#ifdef QM_CHECKING 923296177Sjhibbits register struct qm_mr *mr = &portal->mr; 924296177Sjhibbits ASSERT_COND(mr->pmode == e_QmPortalPCE); 925296177Sjhibbits#endif /* QM_CHECKING */ 926296177Sjhibbits qm_cl_invalidate(MR_PI); 927296177Sjhibbits qm_cl_touch_ro(MR_PI); 928296177Sjhibbits} 929296177Sjhibbits 930296177Sjhibbitsstatic __inline__ uint8_t qmPortalMrPceUpdate(struct qm_portal *portal) 931296177Sjhibbits{ 932296177Sjhibbits register struct qm_mr *mr = &portal->mr; 933296177Sjhibbits uint8_t diff, old_pi = mr->pi; 934296177Sjhibbits#ifdef QM_CHECKING 935296177Sjhibbits ASSERT_COND(mr->pmode == e_QmPortalPCE); 936296177Sjhibbits#endif /* QM_CHECKING */ 937296177Sjhibbits mr->pi = (uint8_t)(qm_cl_in(MR_PI) & (QM_MR_SIZE - 1)); 938296177Sjhibbits diff = cyc_diff(QM_MR_SIZE, old_pi, mr->pi); 939296177Sjhibbits mr->fill += diff; 940296177Sjhibbits return diff; 941296177Sjhibbits} 942296177Sjhibbits 943296177Sjhibbitsstatic __inline__ void qmPortalMrPvbUpdate(struct qm_portal *portal) 944296177Sjhibbits{ 945296177Sjhibbits register struct qm_mr *mr = &portal->mr; 946296177Sjhibbits struct qm_mr_entry *res = ptr_ADD(mr->ring, qm_cl(mr->pi)); 947296177Sjhibbits#ifdef QM_CHECKING 948296177Sjhibbits ASSERT_COND(mr->pmode == e_QmPortalPVB); 949296177Sjhibbits#endif /* QM_CHECKING */ 950296177Sjhibbits dcbit_ro(ptr_ADD(mr->ring, qm_cl(mr->pi))); 951296177Sjhibbits if ((res->verb & QM_MR_VERB_VBIT) == mr->vbit) { 952296177Sjhibbits mr->pi = (uint8_t)((mr->pi + 1) & (QM_MR_SIZE - 1)); 953296177Sjhibbits if (!mr->pi) 954296177Sjhibbits mr->vbit ^= QM_MR_VERB_VBIT; 955296177Sjhibbits mr->fill++; 956296177Sjhibbits } 957296177Sjhibbits} 958296177Sjhibbits 959296177Sjhibbitsstatic __inline__ void qmPortalMrCciConsume(struct qm_portal *portal, uint8_t num) 960296177Sjhibbits{ 961296177Sjhibbits register struct qm_mr *mr = &portal->mr; 962296177Sjhibbits#ifdef QM_CHECKING 963296177Sjhibbits ASSERT_COND(mr->cmode == e_QmPortalMrCCI); 964296177Sjhibbits#endif /* QM_CHECKING */ 965296177Sjhibbits mr->ci = (uint8_t)((mr->ci + num) & (QM_MR_SIZE - 1)); 966296177Sjhibbits qm_out(MR_CI_CINH, mr->ci); 967296177Sjhibbits} 968296177Sjhibbits 969296177Sjhibbitsstatic __inline__ void qmPortalMrCciConsumeToCurrent(struct qm_portal *portal) 970296177Sjhibbits{ 971296177Sjhibbits register struct qm_mr *mr = &portal->mr; 972296177Sjhibbits#ifdef QM_CHECKING 973296177Sjhibbits ASSERT_COND(mr->cmode == e_QmPortalMrCCI); 974296177Sjhibbits#endif /* QM_CHECKING */ 975296177Sjhibbits mr->ci = MR_PTR2IDX(mr->cursor); 976296177Sjhibbits qm_out(MR_CI_CINH, mr->ci); 977296177Sjhibbits} 978296177Sjhibbits 979296177Sjhibbitsstatic __inline__ void qmPortalMrCcePrefetch(struct qm_portal *portal) 980296177Sjhibbits{ 981296177Sjhibbits#ifdef QM_CHECKING 982296177Sjhibbits register struct qm_mr *mr = &portal->mr; 983296177Sjhibbits ASSERT_COND(mr->cmode == e_QmPortalMrCCE); 984296177Sjhibbits#endif /* QM_CHECKING */ 985296177Sjhibbits qm_cl_invalidate(MR_CI); 986296177Sjhibbits qm_cl_touch_rw(MR_CI); 987296177Sjhibbits} 988296177Sjhibbits 989296177Sjhibbitsstatic __inline__ void qmPortalMrCceConsume(struct qm_portal *portal, uint8_t num) 990296177Sjhibbits{ 991296177Sjhibbits register struct qm_mr *mr = &portal->mr; 992296177Sjhibbits#ifdef QM_CHECKING 993296177Sjhibbits ASSERT_COND(mr->cmode == e_QmPortalMrCCE); 994296177Sjhibbits#endif /* QM_CHECKING */ 995296177Sjhibbits mr->ci = (uint8_t)((mr->ci + num) & (QM_MR_SIZE - 1)); 996296177Sjhibbits qm_cl_out(MR_CI, mr->ci); 997296177Sjhibbits} 998296177Sjhibbits 999296177Sjhibbitsstatic __inline__ void qmPortalMrCceConsumeToCurrent(struct qm_portal *portal) 1000296177Sjhibbits{ 1001296177Sjhibbits register struct qm_mr *mr = &portal->mr; 1002296177Sjhibbits#ifdef QM_CHECKING 1003296177Sjhibbits ASSERT_COND(mr->cmode == e_QmPortalMrCCE); 1004296177Sjhibbits#endif /* QM_CHECKING */ 1005296177Sjhibbits mr->ci = MR_PTR2IDX(mr->cursor); 1006296177Sjhibbits qm_cl_out(MR_CI, mr->ci); 1007296177Sjhibbits} 1008296177Sjhibbits 1009296177Sjhibbitsstatic __inline__ uint8_t qm_mr_get_ci(struct qm_portal *portal) 1010296177Sjhibbits{ 1011296177Sjhibbits register struct qm_mr *mr = &portal->mr; 1012296177Sjhibbits return mr->ci; 1013296177Sjhibbits} 1014296177Sjhibbits 1015296177Sjhibbitsstatic __inline__ uint8_t qm_mr_get_ithresh(struct qm_portal *portal) 1016296177Sjhibbits{ 1017296177Sjhibbits register struct qm_mr *mr = &portal->mr; 1018296177Sjhibbits return mr->ithresh; 1019296177Sjhibbits} 1020296177Sjhibbits 1021296177Sjhibbitsstatic __inline__ void qm_mr_set_ithresh(struct qm_portal *portal, uint8_t ithresh) 1022296177Sjhibbits{ 1023296177Sjhibbits qm_out(MR_ITR, ithresh); 1024296177Sjhibbits} 1025296177Sjhibbits 1026296177Sjhibbits/* ------------------------------ */ 1027296177Sjhibbits/* --- Management command API --- */ 1028296177Sjhibbits 1029296177Sjhibbits/* It's safer to code in terms of the 'mc' object than the 'portal' object, 1030296177Sjhibbits * because the latter runs the risk of copy-n-paste errors from other code where 1031296177Sjhibbits * we could manipulate some other structure within 'portal'. */ 1032296177Sjhibbits/* #define MC_API_START() register struct qm_mc *mc = &portal->mc */ 1033296177Sjhibbits 1034296177Sjhibbitsstatic __inline__ t_Error qm_mc_init(struct qm_portal *portal) 1035296177Sjhibbits{ 1036296177Sjhibbits register struct qm_mc *mc = &portal->mc; 1037296177Sjhibbits if (__qm_portal_bind(portal, QM_BIND_MC)) 1038296177Sjhibbits return ERROR_CODE(E_BUSY); 1039296177Sjhibbits mc->cr = ptr_ADD(portal->addr.addr_ce, CL_CR); 1040296177Sjhibbits mc->rr = ptr_ADD(portal->addr.addr_ce, CL_RR0); 1041296177Sjhibbits mc->rridx = (uint8_t)((mc->cr->__dont_write_directly__verb & QM_MCC_VERB_VBIT) ? 1042296177Sjhibbits 0 : 1); 1043296177Sjhibbits mc->vbit = (uint8_t)(mc->rridx ? QM_MCC_VERB_VBIT : 0); 1044296177Sjhibbits#ifdef QM_CHECKING 1045296177Sjhibbits mc->state = mc_idle; 1046296177Sjhibbits#endif /* QM_CHECKING */ 1047296177Sjhibbits return E_OK; 1048296177Sjhibbits} 1049296177Sjhibbits 1050296177Sjhibbitsstatic __inline__ void qm_mc_finish(struct qm_portal *portal) 1051296177Sjhibbits{ 1052296177Sjhibbits#ifdef QM_CHECKING 1053296177Sjhibbits register struct qm_mc *mc = &portal->mc; 1054296177Sjhibbits ASSERT_COND(mc->state == mc_idle); 1055296177Sjhibbits if (mc->state != mc_idle) 1056296177Sjhibbits REPORT_ERROR(WARNING, E_INVALID_STATE, ("Losing incomplete MC command")); 1057296177Sjhibbits#endif /* QM_CHECKING */ 1058296177Sjhibbits __qm_portal_unbind(portal, QM_BIND_MC); 1059296177Sjhibbits} 1060296177Sjhibbits 1061296177Sjhibbitsstatic __inline__ struct qm_mc_command *qm_mc_start(struct qm_portal *portal) 1062296177Sjhibbits{ 1063296177Sjhibbits register struct qm_mc *mc = &portal->mc; 1064296177Sjhibbits#ifdef QM_CHECKING 1065296177Sjhibbits ASSERT_COND(mc->state == mc_idle); 1066296177Sjhibbits mc->state = mc_user; 1067296177Sjhibbits#endif /* QM_CHECKING */ 1068296177Sjhibbits dcbz_64(mc->cr); 1069296177Sjhibbits return mc->cr; 1070296177Sjhibbits} 1071296177Sjhibbits 1072296177Sjhibbitsstatic __inline__ void qm_mc_abort(struct qm_portal *portal) 1073296177Sjhibbits{ 1074296177Sjhibbits#ifdef QM_CHECKING 1075296177Sjhibbits register struct qm_mc *mc = &portal->mc; 1076296177Sjhibbits ASSERT_COND(mc->state == mc_user); 1077296177Sjhibbits mc->state = mc_idle; 1078296177Sjhibbits#else 1079296177Sjhibbits UNUSED(portal); 1080296177Sjhibbits#endif /* QM_CHECKING */ 1081296177Sjhibbits} 1082296177Sjhibbits 1083296177Sjhibbitsstatic __inline__ void qm_mc_commit(struct qm_portal *portal, uint8_t myverb) 1084296177Sjhibbits{ 1085296177Sjhibbits register struct qm_mc *mc = &portal->mc; 1086296177Sjhibbits#ifdef QM_CHECKING 1087296177Sjhibbits ASSERT_COND(mc->state == mc_user); 1088296177Sjhibbits#endif /* QM_CHECKING */ 1089296177Sjhibbits lwsync(); 1090296177Sjhibbits mc->cr->__dont_write_directly__verb = (uint8_t)(myverb | mc->vbit); 1091296177Sjhibbits dcbf_64(mc->cr); 1092296177Sjhibbits dcbit_ro(mc->rr + mc->rridx); 1093296177Sjhibbits#ifdef QM_CHECKING 1094296177Sjhibbits mc->state = mc_hw; 1095296177Sjhibbits#endif /* QM_CHECKING */ 1096296177Sjhibbits} 1097296177Sjhibbits 1098296177Sjhibbitsstatic __inline__ struct qm_mc_result *qm_mc_result(struct qm_portal *portal) 1099296177Sjhibbits{ 1100296177Sjhibbits register struct qm_mc *mc = &portal->mc; 1101296177Sjhibbits struct qm_mc_result *rr = mc->rr + mc->rridx; 1102296177Sjhibbits#ifdef QM_CHECKING 1103296177Sjhibbits ASSERT_COND(mc->state == mc_hw); 1104296177Sjhibbits#endif /* QM_CHECKING */ 1105296177Sjhibbits /* The inactive response register's verb byte always returns zero until 1106296177Sjhibbits * its command is submitted and completed. This includes the valid-bit, 1107296177Sjhibbits * in case you were wondering... */ 1108296177Sjhibbits if (!rr->verb) { 1109296177Sjhibbits dcbit_ro(rr); 1110296177Sjhibbits return NULL; 1111296177Sjhibbits } 1112296177Sjhibbits mc->rridx ^= 1; 1113296177Sjhibbits mc->vbit ^= QM_MCC_VERB_VBIT; 1114296177Sjhibbits#ifdef QM_CHECKING 1115296177Sjhibbits mc->state = mc_idle; 1116296177Sjhibbits#endif /* QM_CHECKING */ 1117296177Sjhibbits return rr; 1118296177Sjhibbits} 1119296177Sjhibbits 1120296177Sjhibbits/* ------------------------------------- */ 1121296177Sjhibbits/* --- Portal interrupt register API --- */ 1122296177Sjhibbits 1123296177Sjhibbitsstatic __inline__ t_Error qm_isr_init(struct qm_portal *portal) 1124296177Sjhibbits{ 1125296177Sjhibbits if (__qm_portal_bind(portal, QM_BIND_ISR)) 1126296177Sjhibbits return ERROR_CODE(E_BUSY); 1127296177Sjhibbits return E_OK; 1128296177Sjhibbits} 1129296177Sjhibbits 1130296177Sjhibbitsstatic __inline__ void qm_isr_finish(struct qm_portal *portal) 1131296177Sjhibbits{ 1132296177Sjhibbits __qm_portal_unbind(portal, QM_BIND_ISR); 1133296177Sjhibbits} 1134296177Sjhibbits 1135296177Sjhibbitsstatic __inline__ void qm_isr_set_iperiod(struct qm_portal *portal, uint16_t iperiod) 1136296177Sjhibbits{ 1137296177Sjhibbits qm_out(ITPR, iperiod); 1138296177Sjhibbits} 1139296177Sjhibbits 1140296177Sjhibbitsstatic __inline__ uint32_t __qm_isr_read(struct qm_portal *portal, enum qm_isr_reg n) 1141296177Sjhibbits{ 1142296177Sjhibbits return __qm_in(&portal->addr, PTR_MOVE(REG_ISR, (n << 2))); 1143296177Sjhibbits} 1144296177Sjhibbits 1145296177Sjhibbitsstatic __inline__ void __qm_isr_write(struct qm_portal *portal, enum qm_isr_reg n, uint32_t val) 1146296177Sjhibbits{ 1147296177Sjhibbits __qm_out(&portal->addr, PTR_MOVE(REG_ISR, (n << 2)), val); 1148296177Sjhibbits} 1149