1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2/* 3 * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. 4 * Copyright 2016-2019 NXP 5 * 6 */ 7 8#include <asm/cacheflush.h> 9#include <linux/io.h> 10#include <linux/slab.h> 11#include <linux/spinlock.h> 12#include <soc/fsl/dpaa2-global.h> 13 14#include "qbman-portal.h" 15 16/* All QBMan command and result structures use this "valid bit" encoding */ 17#define QB_VALID_BIT ((u32)0x80) 18 19/* QBMan portal management command codes */ 20#define QBMAN_MC_ACQUIRE 0x30 21#define QBMAN_WQCHAN_CONFIGURE 0x46 22 23/* CINH register offsets */ 24#define QBMAN_CINH_SWP_EQCR_PI 0x800 25#define QBMAN_CINH_SWP_EQCR_CI 0x840 26#define QBMAN_CINH_SWP_EQAR 0x8c0 27#define QBMAN_CINH_SWP_CR_RT 0x900 28#define QBMAN_CINH_SWP_VDQCR_RT 0x940 29#define QBMAN_CINH_SWP_EQCR_AM_RT 0x980 30#define QBMAN_CINH_SWP_RCR_AM_RT 0x9c0 31#define QBMAN_CINH_SWP_DQPI 0xa00 32#define QBMAN_CINH_SWP_DQRR_ITR 0xa80 33#define QBMAN_CINH_SWP_DCAP 0xac0 34#define QBMAN_CINH_SWP_SDQCR 0xb00 35#define QBMAN_CINH_SWP_EQCR_AM_RT2 0xb40 36#define QBMAN_CINH_SWP_RCR_PI 0xc00 37#define QBMAN_CINH_SWP_RAR 0xcc0 38#define QBMAN_CINH_SWP_ISR 0xe00 39#define QBMAN_CINH_SWP_IER 0xe40 40#define QBMAN_CINH_SWP_ISDR 0xe80 41#define QBMAN_CINH_SWP_IIR 0xec0 42#define QBMAN_CINH_SWP_ITPR 0xf40 43 44/* CENA register offsets */ 45#define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((u32)(n) << 6)) 46#define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((u32)(n) << 6)) 47#define QBMAN_CENA_SWP_RCR(n) (0x400 + ((u32)(n) << 6)) 48#define QBMAN_CENA_SWP_CR 0x600 49#define QBMAN_CENA_SWP_RR(vb) (0x700 + ((u32)(vb) >> 1)) 50#define QBMAN_CENA_SWP_VDQCR 0x780 51#define QBMAN_CENA_SWP_EQCR_CI 0x840 52#define QBMAN_CENA_SWP_EQCR_CI_MEMBACK 0x1840 53 54/* CENA register offsets in memory-backed mode */ 55#define QBMAN_CENA_SWP_DQRR_MEM(n) (0x800 + ((u32)(n) << 6)) 56#define QBMAN_CENA_SWP_RCR_MEM(n) (0x1400 + ((u32)(n) << 6)) 57#define QBMAN_CENA_SWP_CR_MEM 0x1600 58#define QBMAN_CENA_SWP_RR_MEM 0x1680 59#define QBMAN_CENA_SWP_VDQCR_MEM 0x1780 60 61/* Reverse mapping of QBMAN_CENA_SWP_DQRR() */ 62#define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)(p) & 0x1ff) >> 6) 63 64/* Define token used to determine if response written to memory is valid */ 65#define QMAN_DQ_TOKEN_VALID 1 66 67/* SDQCR attribute codes */ 68#define QB_SDQCR_FC_SHIFT 29 69#define QB_SDQCR_FC_MASK 0x1 70#define QB_SDQCR_DCT_SHIFT 24 71#define QB_SDQCR_DCT_MASK 0x3 72#define QB_SDQCR_TOK_SHIFT 16 73#define QB_SDQCR_TOK_MASK 0xff 74#define QB_SDQCR_SRC_SHIFT 0 75#define QB_SDQCR_SRC_MASK 0xffff 76 77/* opaque token for static dequeues */ 78#define QMAN_SDQCR_TOKEN 0xbb 79 80#define QBMAN_EQCR_DCA_IDXMASK 0x0f 81#define QBMAN_ENQUEUE_FLAG_DCA (1ULL << 31) 82 83#define EQ_DESC_SIZE_WITHOUT_FD 29 84#define EQ_DESC_SIZE_FD_START 32 85 86enum qbman_sdqcr_dct { 87 qbman_sdqcr_dct_null = 0, 88 qbman_sdqcr_dct_prio_ics, 89 qbman_sdqcr_dct_active_ics, 90 qbman_sdqcr_dct_active 91}; 92 93enum qbman_sdqcr_fc { 94 qbman_sdqcr_fc_one = 0, 95 qbman_sdqcr_fc_up_to_3 = 1 96}; 97 98/* Internal Function declaration */ 99static int qbman_swp_enqueue_direct(struct qbman_swp *s, 100 const struct qbman_eq_desc *d, 101 const struct dpaa2_fd *fd); 102static int qbman_swp_enqueue_mem_back(struct qbman_swp *s, 103 const struct qbman_eq_desc *d, 104 const struct dpaa2_fd *fd); 105static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, 106 const struct qbman_eq_desc *d, 107 const struct dpaa2_fd *fd, 108 uint32_t *flags, 109 int num_frames); 110static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s, 111 const struct qbman_eq_desc *d, 112 const struct dpaa2_fd *fd, 113 uint32_t *flags, 114 int num_frames); 115static int 116qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s, 117 const struct qbman_eq_desc *d, 118 const struct dpaa2_fd *fd, 119 int num_frames); 120static 121int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s, 122 const struct qbman_eq_desc *d, 123 const struct dpaa2_fd *fd, 124 int num_frames); 125static int qbman_swp_pull_direct(struct qbman_swp *s, 126 struct qbman_pull_desc *d); 127static int qbman_swp_pull_mem_back(struct qbman_swp *s, 128 struct qbman_pull_desc *d); 129 130const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s); 131const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s); 132 133static int qbman_swp_release_direct(struct qbman_swp *s, 134 const struct qbman_release_desc *d, 135 const u64 *buffers, 136 unsigned int num_buffers); 137static int qbman_swp_release_mem_back(struct qbman_swp *s, 138 const struct qbman_release_desc *d, 139 const u64 *buffers, 140 unsigned int num_buffers); 141 142/* Function pointers */ 143int (*qbman_swp_enqueue_ptr)(struct qbman_swp *s, 144 const struct qbman_eq_desc *d, 145 const struct dpaa2_fd *fd) 146 = qbman_swp_enqueue_direct; 147 148int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s, 149 const struct qbman_eq_desc *d, 150 const struct dpaa2_fd *fd, 151 uint32_t *flags, 152 int num_frames) 153 = qbman_swp_enqueue_multiple_direct; 154 155int 156(*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s, 157 const struct qbman_eq_desc *d, 158 const struct dpaa2_fd *fd, 159 int num_frames) 160 = qbman_swp_enqueue_multiple_desc_direct; 161 162int (*qbman_swp_pull_ptr)(struct qbman_swp *s, struct qbman_pull_desc *d) 163 = qbman_swp_pull_direct; 164 165const struct dpaa2_dq *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s) 166 = qbman_swp_dqrr_next_direct; 167 168int (*qbman_swp_release_ptr)(struct qbman_swp *s, 169 const struct qbman_release_desc *d, 170 const u64 *buffers, 171 unsigned int num_buffers) 172 = qbman_swp_release_direct; 173 174/* Portal Access */ 175 176static inline u32 qbman_read_register(struct qbman_swp *p, u32 offset) 177{ 178 return readl_relaxed(p->addr_cinh + offset); 179} 180 181static inline void qbman_write_register(struct qbman_swp *p, u32 offset, 182 u32 value) 183{ 184 writel_relaxed(value, p->addr_cinh + offset); 185} 186 187static inline void *qbman_get_cmd(struct qbman_swp *p, u32 offset) 188{ 189 return p->addr_cena + offset; 190} 191 192#define QBMAN_CINH_SWP_CFG 0xd00 193 194#define SWP_CFG_DQRR_MF_SHIFT 20 195#define SWP_CFG_EST_SHIFT 16 196#define SWP_CFG_CPBS_SHIFT 15 197#define SWP_CFG_WN_SHIFT 14 198#define SWP_CFG_RPM_SHIFT 12 199#define SWP_CFG_DCM_SHIFT 10 200#define SWP_CFG_EPM_SHIFT 8 201#define SWP_CFG_VPM_SHIFT 7 202#define SWP_CFG_CPM_SHIFT 6 203#define SWP_CFG_SD_SHIFT 5 204#define SWP_CFG_SP_SHIFT 4 205#define SWP_CFG_SE_SHIFT 3 206#define SWP_CFG_DP_SHIFT 2 207#define SWP_CFG_DE_SHIFT 1 208#define SWP_CFG_EP_SHIFT 0 209 210static inline u32 qbman_set_swp_cfg(u8 max_fill, u8 wn, u8 est, u8 rpm, u8 dcm, 211 u8 epm, int sd, int sp, int se, 212 int dp, int de, int ep) 213{ 214 return (max_fill << SWP_CFG_DQRR_MF_SHIFT | 215 est << SWP_CFG_EST_SHIFT | 216 wn << SWP_CFG_WN_SHIFT | 217 rpm << SWP_CFG_RPM_SHIFT | 218 dcm << SWP_CFG_DCM_SHIFT | 219 epm << SWP_CFG_EPM_SHIFT | 220 sd << SWP_CFG_SD_SHIFT | 221 sp << SWP_CFG_SP_SHIFT | 222 se << SWP_CFG_SE_SHIFT | 223 dp << SWP_CFG_DP_SHIFT | 224 de << SWP_CFG_DE_SHIFT | 225 ep << SWP_CFG_EP_SHIFT); 226} 227 228#define QMAN_RT_MODE 0x00000100 229 230static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last) 231{ 232 /* 'first' is included, 'last' is excluded */ 233 if (first <= last) 234 return last - first; 235 else 236 return (2 * ringsize) - (first - last); 237} 238 239/** 240 * qbman_swp_init() - Create a functional object representing the given 241 * QBMan portal descriptor. 242 * @d: the given qbman swp descriptor 243 * 244 * Return qbman_swp portal for success, NULL if the object cannot 245 * be created. 246 */ 247struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) 248{ 249 struct qbman_swp *p = kzalloc(sizeof(*p), GFP_KERNEL); 250 u32 reg; 251 u32 mask_size; 252 u32 eqcr_pi; 253 254 if (!p) 255 return NULL; 256 257 spin_lock_init(&p->access_spinlock); 258 259 p->desc = d; 260 p->mc.valid_bit = QB_VALID_BIT; 261 p->sdq = 0; 262 p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT; 263 p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT; 264 p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT; 265 if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) 266 p->mr.valid_bit = QB_VALID_BIT; 267 268 atomic_set(&p->vdq.available, 1); 269 p->vdq.valid_bit = QB_VALID_BIT; 270 p->dqrr.next_idx = 0; 271 p->dqrr.valid_bit = QB_VALID_BIT; 272 273 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_4100) { 274 p->dqrr.dqrr_size = 4; 275 p->dqrr.reset_bug = 1; 276 } else { 277 p->dqrr.dqrr_size = 8; 278 p->dqrr.reset_bug = 0; 279 } 280 281 p->addr_cena = d->cena_bar; 282 p->addr_cinh = d->cinh_bar; 283 284 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { 285 286 reg = qbman_set_swp_cfg(p->dqrr.dqrr_size, 287 1, /* Writes Non-cacheable */ 288 0, /* EQCR_CI stashing threshold */ 289 3, /* RPM: RCR in array mode */ 290 2, /* DCM: Discrete consumption ack */ 291 2, /* EPM: EQCR in ring mode */ 292 1, /* mem stashing drop enable enable */ 293 1, /* mem stashing priority enable */ 294 1, /* mem stashing enable */ 295 1, /* dequeue stashing priority enable */ 296 0, /* dequeue stashing enable enable */ 297 0); /* EQCR_CI stashing priority enable */ 298 } else { 299 memset(p->addr_cena, 0, 64 * 1024); 300 reg = qbman_set_swp_cfg(p->dqrr.dqrr_size, 301 1, /* Writes Non-cacheable */ 302 1, /* EQCR_CI stashing threshold */ 303 3, /* RPM: RCR in array mode */ 304 2, /* DCM: Discrete consumption ack */ 305 0, /* EPM: EQCR in ring mode */ 306 1, /* mem stashing drop enable */ 307 1, /* mem stashing priority enable */ 308 1, /* mem stashing enable */ 309 1, /* dequeue stashing priority enable */ 310 0, /* dequeue stashing enable */ 311 0); /* EQCR_CI stashing priority enable */ 312 reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */ 313 1 << SWP_CFG_VPM_SHIFT | /* VDQCR read triggered mode */ 314 1 << SWP_CFG_CPM_SHIFT; /* CR read triggered mode */ 315 } 316 317 qbman_write_register(p, QBMAN_CINH_SWP_CFG, reg); 318 reg = qbman_read_register(p, QBMAN_CINH_SWP_CFG); 319 if (!reg) { 320 pr_err("qbman: the portal is not enabled!\n"); 321 kfree(p); 322 return NULL; 323 } 324 325 if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) { 326 qbman_write_register(p, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE); 327 qbman_write_register(p, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE); 328 } 329 /* 330 * SDQCR needs to be initialized to 0 when no channels are 331 * being dequeued from or else the QMan HW will indicate an 332 * error. The values that were calculated above will be 333 * applied when dequeues from a specific channel are enabled. 334 */ 335 qbman_write_register(p, QBMAN_CINH_SWP_SDQCR, 0); 336 337 p->eqcr.pi_ring_size = 8; 338 if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) { 339 p->eqcr.pi_ring_size = 32; 340 qbman_swp_enqueue_ptr = 341 qbman_swp_enqueue_mem_back; 342 qbman_swp_enqueue_multiple_ptr = 343 qbman_swp_enqueue_multiple_mem_back; 344 qbman_swp_enqueue_multiple_desc_ptr = 345 qbman_swp_enqueue_multiple_desc_mem_back; 346 qbman_swp_pull_ptr = qbman_swp_pull_mem_back; 347 qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back; 348 qbman_swp_release_ptr = qbman_swp_release_mem_back; 349 } 350 351 for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1) 352 p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1; 353 eqcr_pi = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_PI); 354 p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask; 355 p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT; 356 p->eqcr.ci = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_CI) 357 & p->eqcr.pi_ci_mask; 358 p->eqcr.available = p->eqcr.pi_ring_size; 359 360 /* Initialize the software portal with a irq timeout period of 0us */ 361 qbman_swp_set_irq_coalescing(p, p->dqrr.dqrr_size - 1, 0); 362 363 return p; 364} 365 366/** 367 * qbman_swp_finish() - Create and destroy a functional object representing 368 * the given QBMan portal descriptor. 369 * @p: the qbman_swp object to be destroyed 370 */ 371void qbman_swp_finish(struct qbman_swp *p) 372{ 373 kfree(p); 374} 375 376/** 377 * qbman_swp_interrupt_read_status() 378 * @p: the given software portal 379 * 380 * Return the value in the SWP_ISR register. 381 */ 382u32 qbman_swp_interrupt_read_status(struct qbman_swp *p) 383{ 384 return qbman_read_register(p, QBMAN_CINH_SWP_ISR); 385} 386 387/** 388 * qbman_swp_interrupt_clear_status() 389 * @p: the given software portal 390 * @mask: The mask to clear in SWP_ISR register 391 */ 392void qbman_swp_interrupt_clear_status(struct qbman_swp *p, u32 mask) 393{ 394 qbman_write_register(p, QBMAN_CINH_SWP_ISR, mask); 395} 396 397/** 398 * qbman_swp_interrupt_get_trigger() - read interrupt enable register 399 * @p: the given software portal 400 * 401 * Return the value in the SWP_IER register. 402 */ 403u32 qbman_swp_interrupt_get_trigger(struct qbman_swp *p) 404{ 405 return qbman_read_register(p, QBMAN_CINH_SWP_IER); 406} 407 408/** 409 * qbman_swp_interrupt_set_trigger() - enable interrupts for a swp 410 * @p: the given software portal 411 * @mask: The mask of bits to enable in SWP_IER 412 */ 413void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, u32 mask) 414{ 415 qbman_write_register(p, QBMAN_CINH_SWP_IER, mask); 416} 417 418/** 419 * qbman_swp_interrupt_get_inhibit() - read interrupt mask register 420 * @p: the given software portal object 421 * 422 * Return the value in the SWP_IIR register. 423 */ 424int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p) 425{ 426 return qbman_read_register(p, QBMAN_CINH_SWP_IIR); 427} 428 429/** 430 * qbman_swp_interrupt_set_inhibit() - write interrupt mask register 431 * @p: the given software portal object 432 * @inhibit: whether to inhibit the IRQs 433 */ 434void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit) 435{ 436 qbman_write_register(p, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0); 437} 438 439/* 440 * Different management commands all use this common base layer of code to issue 441 * commands and poll for results. 442 */ 443 444/* 445 * Returns a pointer to where the caller should fill in their management command 446 * (caller should ignore the verb byte) 447 */ 448void *qbman_swp_mc_start(struct qbman_swp *p) 449{ 450 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) 451 return qbman_get_cmd(p, QBMAN_CENA_SWP_CR); 452 else 453 return qbman_get_cmd(p, QBMAN_CENA_SWP_CR_MEM); 454} 455 456/* 457 * Commits merges in the caller-supplied command verb (which should not include 458 * the valid-bit) and submits the command to hardware 459 */ 460void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, u8 cmd_verb) 461{ 462 u8 *v = cmd; 463 464 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { 465 dma_wmb(); 466 *v = cmd_verb | p->mc.valid_bit; 467 } else { 468 *v = cmd_verb | p->mc.valid_bit; 469 dma_wmb(); 470 qbman_write_register(p, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE); 471 } 472} 473 474/* 475 * Checks for a completed response (returns non-NULL if only if the response 476 * is complete). 477 */ 478void *qbman_swp_mc_result(struct qbman_swp *p) 479{ 480 u32 *ret, verb; 481 482 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { 483 ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR(p->mc.valid_bit)); 484 /* Remove the valid-bit - command completed if the rest 485 * is non-zero. 486 */ 487 verb = ret[0] & ~QB_VALID_BIT; 488 if (!verb) 489 return NULL; 490 p->mc.valid_bit ^= QB_VALID_BIT; 491 } else { 492 ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR_MEM); 493 /* Command completed if the valid bit is toggled */ 494 if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT)) 495 return NULL; 496 /* Command completed if the rest is non-zero */ 497 verb = ret[0] & ~QB_VALID_BIT; 498 if (!verb) 499 return NULL; 500 p->mr.valid_bit ^= QB_VALID_BIT; 501 } 502 503 return ret; 504} 505 506#define QB_ENQUEUE_CMD_OPTIONS_SHIFT 0 507enum qb_enqueue_commands { 508 enqueue_empty = 0, 509 enqueue_response_always = 1, 510 enqueue_rejects_to_fq = 2 511}; 512 513#define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT 2 514#define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3 515#define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT 4 516#define QB_ENQUEUE_CMD_DCA_EN_SHIFT 7 517 518/* 519 * qbman_eq_desc_clear() - Clear the contents of a descriptor to 520 * default/starting state. 521 */ 522void qbman_eq_desc_clear(struct qbman_eq_desc *d) 523{ 524 memset(d, 0, sizeof(*d)); 525} 526 527/** 528 * qbman_eq_desc_set_no_orp() - Set enqueue descriptor without orp 529 * @d: the enqueue descriptor. 530 * @respond_success: 1 = enqueue with response always; 0 = enqueue with 531 * rejections returned on a FQ. 532 */ 533void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success) 534{ 535 d->verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT); 536 if (respond_success) 537 d->verb |= enqueue_response_always; 538 else 539 d->verb |= enqueue_rejects_to_fq; 540} 541 542/* 543 * Exactly one of the following descriptor "targets" should be set. (Calling any 544 * one of these will replace the effect of any prior call to one of these.) 545 * -enqueue to a frame queue 546 * -enqueue to a queuing destination 547 */ 548 549/** 550 * qbman_eq_desc_set_fq() - set the FQ for the enqueue command 551 * @d: the enqueue descriptor 552 * @fqid: the id of the frame queue to be enqueued 553 */ 554void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, u32 fqid) 555{ 556 d->verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT); 557 d->tgtid = cpu_to_le32(fqid); 558} 559 560/** 561 * qbman_eq_desc_set_qd() - Set Queuing Destination for the enqueue command 562 * @d: the enqueue descriptor 563 * @qdid: the id of the queuing destination to be enqueued 564 * @qd_bin: the queuing destination bin 565 * @qd_prio: the queuing destination priority 566 */ 567void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, u32 qdid, 568 u32 qd_bin, u32 qd_prio) 569{ 570 d->verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT; 571 d->tgtid = cpu_to_le32(qdid); 572 d->qdbin = cpu_to_le16(qd_bin); 573 d->qpri = qd_prio; 574} 575 576#define EQAR_IDX(eqar) ((eqar) & 0x7) 577#define EQAR_VB(eqar) ((eqar) & 0x80) 578#define EQAR_SUCCESS(eqar) ((eqar) & 0x100) 579 580#define QB_RT_BIT ((u32)0x100) 581/** 582 * qbman_swp_enqueue_direct() - Issue an enqueue command 583 * @s: the software portal used for enqueue 584 * @d: the enqueue descriptor 585 * @fd: the frame descriptor to be enqueued 586 * 587 * Please note that 'fd' should only be NULL if the "action" of the 588 * descriptor is "orp_hole" or "orp_nesn". 589 * 590 * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready. 591 */ 592static 593int qbman_swp_enqueue_direct(struct qbman_swp *s, 594 const struct qbman_eq_desc *d, 595 const struct dpaa2_fd *fd) 596{ 597 int flags = 0; 598 int ret = qbman_swp_enqueue_multiple_direct(s, d, fd, &flags, 1); 599 600 if (ret >= 0) 601 ret = 0; 602 else 603 ret = -EBUSY; 604 return ret; 605} 606 607/** 608 * qbman_swp_enqueue_mem_back() - Issue an enqueue command 609 * @s: the software portal used for enqueue 610 * @d: the enqueue descriptor 611 * @fd: the frame descriptor to be enqueued 612 * 613 * Please note that 'fd' should only be NULL if the "action" of the 614 * descriptor is "orp_hole" or "orp_nesn". 615 * 616 * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready. 617 */ 618static 619int qbman_swp_enqueue_mem_back(struct qbman_swp *s, 620 const struct qbman_eq_desc *d, 621 const struct dpaa2_fd *fd) 622{ 623 int flags = 0; 624 int ret = qbman_swp_enqueue_multiple_mem_back(s, d, fd, &flags, 1); 625 626 if (ret >= 0) 627 ret = 0; 628 else 629 ret = -EBUSY; 630 return ret; 631} 632 633/** 634 * qbman_swp_enqueue_multiple_direct() - Issue a multi enqueue command 635 * using one enqueue descriptor 636 * @s: the software portal used for enqueue 637 * @d: the enqueue descriptor 638 * @fd: table pointer of frame descriptor table to be enqueued 639 * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL 640 * @num_frames: number of fd to be enqueued 641 * 642 * Return the number of fd enqueued, or a negative error number. 643 */ 644static 645int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, 646 const struct qbman_eq_desc *d, 647 const struct dpaa2_fd *fd, 648 uint32_t *flags, 649 int num_frames) 650{ 651 uint32_t *p = NULL; 652 const uint32_t *cl = (uint32_t *)d; 653 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 654 int i, num_enqueued = 0; 655 656 spin_lock(&s->access_spinlock); 657 half_mask = (s->eqcr.pi_ci_mask>>1); 658 full_mask = s->eqcr.pi_ci_mask; 659 660 if (!s->eqcr.available) { 661 eqcr_ci = s->eqcr.ci; 662 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI; 663 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI); 664 s->eqcr.ci &= full_mask; 665 666 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 667 eqcr_ci, s->eqcr.ci); 668 if (!s->eqcr.available) { 669 spin_unlock(&s->access_spinlock); 670 return 0; 671 } 672 } 673 674 eqcr_pi = s->eqcr.pi; 675 num_enqueued = (s->eqcr.available < num_frames) ? 676 s->eqcr.available : num_frames; 677 s->eqcr.available -= num_enqueued; 678 /* Fill in the EQCR ring */ 679 for (i = 0; i < num_enqueued; i++) { 680 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 681 /* Skip copying the verb */ 682 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); 683 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)], 684 &fd[i], sizeof(*fd)); 685 eqcr_pi++; 686 } 687 688 dma_wmb(); 689 690 /* Set the verb byte, have to substitute in the valid-bit */ 691 eqcr_pi = s->eqcr.pi; 692 for (i = 0; i < num_enqueued; i++) { 693 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 694 p[0] = cl[0] | s->eqcr.pi_vb; 695 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { 696 struct qbman_eq_desc *eq_desc = (struct qbman_eq_desc *)p; 697 698 eq_desc->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | 699 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); 700 } 701 eqcr_pi++; 702 if (!(eqcr_pi & half_mask)) 703 s->eqcr.pi_vb ^= QB_VALID_BIT; 704 } 705 706 /* Flush all the cacheline without load/store in between */ 707 eqcr_pi = s->eqcr.pi; 708 for (i = 0; i < num_enqueued; i++) 709 eqcr_pi++; 710 s->eqcr.pi = eqcr_pi & full_mask; 711 spin_unlock(&s->access_spinlock); 712 713 return num_enqueued; 714} 715 716/** 717 * qbman_swp_enqueue_multiple_mem_back() - Issue a multi enqueue command 718 * using one enqueue descriptor 719 * @s: the software portal used for enqueue 720 * @d: the enqueue descriptor 721 * @fd: table pointer of frame descriptor table to be enqueued 722 * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL 723 * @num_frames: number of fd to be enqueued 724 * 725 * Return the number of fd enqueued, or a negative error number. 726 */ 727static 728int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s, 729 const struct qbman_eq_desc *d, 730 const struct dpaa2_fd *fd, 731 uint32_t *flags, 732 int num_frames) 733{ 734 uint32_t *p = NULL; 735 const uint32_t *cl = (uint32_t *)(d); 736 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 737 int i, num_enqueued = 0; 738 unsigned long irq_flags; 739 740 spin_lock_irqsave(&s->access_spinlock, irq_flags); 741 742 half_mask = (s->eqcr.pi_ci_mask>>1); 743 full_mask = s->eqcr.pi_ci_mask; 744 if (!s->eqcr.available) { 745 eqcr_ci = s->eqcr.ci; 746 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI); 747 s->eqcr.ci &= full_mask; 748 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 749 eqcr_ci, s->eqcr.ci); 750 if (!s->eqcr.available) { 751 spin_unlock_irqrestore(&s->access_spinlock, irq_flags); 752 return 0; 753 } 754 } 755 756 eqcr_pi = s->eqcr.pi; 757 num_enqueued = (s->eqcr.available < num_frames) ? 758 s->eqcr.available : num_frames; 759 s->eqcr.available -= num_enqueued; 760 /* Fill in the EQCR ring */ 761 for (i = 0; i < num_enqueued; i++) { 762 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 763 /* Skip copying the verb */ 764 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); 765 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)], 766 &fd[i], sizeof(*fd)); 767 eqcr_pi++; 768 } 769 770 /* Set the verb byte, have to substitute in the valid-bit */ 771 eqcr_pi = s->eqcr.pi; 772 for (i = 0; i < num_enqueued; i++) { 773 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 774 p[0] = cl[0] | s->eqcr.pi_vb; 775 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { 776 struct qbman_eq_desc *eq_desc = (struct qbman_eq_desc *)p; 777 778 eq_desc->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | 779 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); 780 } 781 eqcr_pi++; 782 if (!(eqcr_pi & half_mask)) 783 s->eqcr.pi_vb ^= QB_VALID_BIT; 784 } 785 s->eqcr.pi = eqcr_pi & full_mask; 786 787 dma_wmb(); 788 qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI, 789 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); 790 spin_unlock_irqrestore(&s->access_spinlock, irq_flags); 791 792 return num_enqueued; 793} 794 795/** 796 * qbman_swp_enqueue_multiple_desc_direct() - Issue a multi enqueue command 797 * using multiple enqueue descriptor 798 * @s: the software portal used for enqueue 799 * @d: table of minimal enqueue descriptor 800 * @fd: table pointer of frame descriptor table to be enqueued 801 * @num_frames: number of fd to be enqueued 802 * 803 * Return the number of fd enqueued, or a negative error number. 804 */ 805static 806int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s, 807 const struct qbman_eq_desc *d, 808 const struct dpaa2_fd *fd, 809 int num_frames) 810{ 811 uint32_t *p; 812 const uint32_t *cl; 813 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 814 int i, num_enqueued = 0; 815 816 half_mask = (s->eqcr.pi_ci_mask>>1); 817 full_mask = s->eqcr.pi_ci_mask; 818 if (!s->eqcr.available) { 819 eqcr_ci = s->eqcr.ci; 820 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI; 821 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI); 822 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 823 eqcr_ci, s->eqcr.ci); 824 if (!s->eqcr.available) 825 return 0; 826 } 827 828 eqcr_pi = s->eqcr.pi; 829 num_enqueued = (s->eqcr.available < num_frames) ? 830 s->eqcr.available : num_frames; 831 s->eqcr.available -= num_enqueued; 832 /* Fill in the EQCR ring */ 833 for (i = 0; i < num_enqueued; i++) { 834 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 835 cl = (uint32_t *)(&d[i]); 836 /* Skip copying the verb */ 837 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); 838 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)], 839 &fd[i], sizeof(*fd)); 840 eqcr_pi++; 841 } 842 843 dma_wmb(); 844 845 /* Set the verb byte, have to substitute in the valid-bit */ 846 eqcr_pi = s->eqcr.pi; 847 for (i = 0; i < num_enqueued; i++) { 848 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 849 cl = (uint32_t *)(&d[i]); 850 p[0] = cl[0] | s->eqcr.pi_vb; 851 eqcr_pi++; 852 if (!(eqcr_pi & half_mask)) 853 s->eqcr.pi_vb ^= QB_VALID_BIT; 854 } 855 856 /* Flush all the cacheline without load/store in between */ 857 eqcr_pi = s->eqcr.pi; 858 for (i = 0; i < num_enqueued; i++) 859 eqcr_pi++; 860 s->eqcr.pi = eqcr_pi & full_mask; 861 862 return num_enqueued; 863} 864 865/** 866 * qbman_swp_enqueue_multiple_desc_mem_back() - Issue a multi enqueue command 867 * using multiple enqueue descriptor 868 * @s: the software portal used for enqueue 869 * @d: table of minimal enqueue descriptor 870 * @fd: table pointer of frame descriptor table to be enqueued 871 * @num_frames: number of fd to be enqueued 872 * 873 * Return the number of fd enqueued, or a negative error number. 874 */ 875static 876int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s, 877 const struct qbman_eq_desc *d, 878 const struct dpaa2_fd *fd, 879 int num_frames) 880{ 881 uint32_t *p; 882 const uint32_t *cl; 883 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 884 int i, num_enqueued = 0; 885 886 half_mask = (s->eqcr.pi_ci_mask>>1); 887 full_mask = s->eqcr.pi_ci_mask; 888 if (!s->eqcr.available) { 889 eqcr_ci = s->eqcr.ci; 890 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI); 891 s->eqcr.ci &= full_mask; 892 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 893 eqcr_ci, s->eqcr.ci); 894 if (!s->eqcr.available) 895 return 0; 896 } 897 898 eqcr_pi = s->eqcr.pi; 899 num_enqueued = (s->eqcr.available < num_frames) ? 900 s->eqcr.available : num_frames; 901 s->eqcr.available -= num_enqueued; 902 /* Fill in the EQCR ring */ 903 for (i = 0; i < num_enqueued; i++) { 904 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 905 cl = (uint32_t *)(&d[i]); 906 /* Skip copying the verb */ 907 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); 908 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)], 909 &fd[i], sizeof(*fd)); 910 eqcr_pi++; 911 } 912 913 /* Set the verb byte, have to substitute in the valid-bit */ 914 eqcr_pi = s->eqcr.pi; 915 for (i = 0; i < num_enqueued; i++) { 916 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 917 cl = (uint32_t *)(&d[i]); 918 p[0] = cl[0] | s->eqcr.pi_vb; 919 eqcr_pi++; 920 if (!(eqcr_pi & half_mask)) 921 s->eqcr.pi_vb ^= QB_VALID_BIT; 922 } 923 924 s->eqcr.pi = eqcr_pi & full_mask; 925 926 dma_wmb(); 927 qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI, 928 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); 929 930 return num_enqueued; 931} 932 933/* Static (push) dequeue */ 934 935/** 936 * qbman_swp_push_get() - Get the push dequeue setup 937 * @s: the software portal object 938 * @channel_idx: the channel index to query 939 * @enabled: returned boolean to show whether the push dequeue is enabled 940 * for the given channel 941 */ 942void qbman_swp_push_get(struct qbman_swp *s, u8 channel_idx, int *enabled) 943{ 944 u16 src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; 945 946 WARN_ON(channel_idx > 15); 947 *enabled = src | (1 << channel_idx); 948} 949 950/** 951 * qbman_swp_push_set() - Enable or disable push dequeue 952 * @s: the software portal object 953 * @channel_idx: the channel index (0 to 15) 954 * @enable: enable or disable push dequeue 955 */ 956void qbman_swp_push_set(struct qbman_swp *s, u8 channel_idx, int enable) 957{ 958 u16 dqsrc; 959 960 WARN_ON(channel_idx > 15); 961 if (enable) 962 s->sdq |= 1 << channel_idx; 963 else 964 s->sdq &= ~(1 << channel_idx); 965 966 /* Read make the complete src map. If no channels are enabled 967 * the SDQCR must be 0 or else QMan will assert errors 968 */ 969 dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; 970 if (dqsrc != 0) 971 qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, s->sdq); 972 else 973 qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, 0); 974} 975 976#define QB_VDQCR_VERB_DCT_SHIFT 0 977#define QB_VDQCR_VERB_DT_SHIFT 2 978#define QB_VDQCR_VERB_RLS_SHIFT 4 979#define QB_VDQCR_VERB_WAE_SHIFT 5 980 981enum qb_pull_dt_e { 982 qb_pull_dt_channel, 983 qb_pull_dt_workqueue, 984 qb_pull_dt_framequeue 985}; 986 987/** 988 * qbman_pull_desc_clear() - Clear the contents of a descriptor to 989 * default/starting state 990 * @d: the pull dequeue descriptor to be cleared 991 */ 992void qbman_pull_desc_clear(struct qbman_pull_desc *d) 993{ 994 memset(d, 0, sizeof(*d)); 995} 996 997/** 998 * qbman_pull_desc_set_storage()- Set the pull dequeue storage 999 * @d: the pull dequeue descriptor to be set 1000 * @storage: the pointer of the memory to store the dequeue result 1001 * @storage_phys: the physical address of the storage memory 1002 * @stash: to indicate whether write allocate is enabled 1003 * 1004 * If not called, or if called with 'storage' as NULL, the result pull dequeues 1005 * will produce results to DQRR. If 'storage' is non-NULL, then results are 1006 * produced to the given memory location (using the DMA address which 1007 * the caller provides in 'storage_phys'), and 'stash' controls whether or not 1008 * those writes to main-memory express a cache-warming attribute. 1009 */ 1010void qbman_pull_desc_set_storage(struct qbman_pull_desc *d, 1011 struct dpaa2_dq *storage, 1012 dma_addr_t storage_phys, 1013 int stash) 1014{ 1015 /* save the virtual address */ 1016 d->rsp_addr_virt = (u64)(uintptr_t)storage; 1017 1018 if (!storage) { 1019 d->verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT); 1020 return; 1021 } 1022 d->verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT; 1023 if (stash) 1024 d->verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT; 1025 else 1026 d->verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT); 1027 1028 d->rsp_addr = cpu_to_le64(storage_phys); 1029} 1030 1031/** 1032 * qbman_pull_desc_set_numframes() - Set the number of frames to be dequeued 1033 * @d: the pull dequeue descriptor to be set 1034 * @numframes: number of frames to be set, must be between 1 and 16, inclusive 1035 */ 1036void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, u8 numframes) 1037{ 1038 d->numf = numframes - 1; 1039} 1040 1041/* 1042 * Exactly one of the following descriptor "actions" should be set. (Calling any 1043 * one of these will replace the effect of any prior call to one of these.) 1044 * - pull dequeue from the given frame queue (FQ) 1045 * - pull dequeue from any FQ in the given work queue (WQ) 1046 * - pull dequeue from any FQ in any WQ in the given channel 1047 */ 1048 1049/** 1050 * qbman_pull_desc_set_fq() - Set fqid from which the dequeue command dequeues 1051 * @d: the pull dequeue descriptor to be set 1052 * @fqid: the frame queue index of the given FQ 1053 */ 1054void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, u32 fqid) 1055{ 1056 d->verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT; 1057 d->verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT; 1058 d->dq_src = cpu_to_le32(fqid); 1059} 1060 1061/** 1062 * qbman_pull_desc_set_wq() - Set wqid from which the dequeue command dequeues 1063 * @d: the pull dequeue descriptor to be set 1064 * @wqid: composed of channel id and wqid within the channel 1065 * @dct: the dequeue command type 1066 */ 1067void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, u32 wqid, 1068 enum qbman_pull_type_e dct) 1069{ 1070 d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; 1071 d->verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT; 1072 d->dq_src = cpu_to_le32(wqid); 1073} 1074 1075/** 1076 * qbman_pull_desc_set_channel() - Set channelid from which the dequeue command 1077 * dequeues 1078 * @d: the pull dequeue descriptor to be set 1079 * @chid: the channel id to be dequeued 1080 * @dct: the dequeue command type 1081 */ 1082void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, u32 chid, 1083 enum qbman_pull_type_e dct) 1084{ 1085 d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; 1086 d->verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT; 1087 d->dq_src = cpu_to_le32(chid); 1088} 1089 1090/** 1091 * qbman_swp_pull_direct() - Issue the pull dequeue command 1092 * @s: the software portal object 1093 * @d: the software portal descriptor which has been configured with 1094 * the set of qbman_pull_desc_set_*() calls 1095 * 1096 * Return 0 for success, and -EBUSY if the software portal is not ready 1097 * to do pull dequeue. 1098 */ 1099static 1100int qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d) 1101{ 1102 struct qbman_pull_desc *p; 1103 1104 if (!atomic_dec_and_test(&s->vdq.available)) { 1105 atomic_inc(&s->vdq.available); 1106 return -EBUSY; 1107 } 1108 s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt; 1109 if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) 1110 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR); 1111 else 1112 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM); 1113 p->numf = d->numf; 1114 p->tok = QMAN_DQ_TOKEN_VALID; 1115 p->dq_src = d->dq_src; 1116 p->rsp_addr = d->rsp_addr; 1117 p->rsp_addr_virt = d->rsp_addr_virt; 1118 dma_wmb(); 1119 /* Set the verb byte, have to substitute in the valid-bit */ 1120 p->verb = d->verb | s->vdq.valid_bit; 1121 s->vdq.valid_bit ^= QB_VALID_BIT; 1122 1123 return 0; 1124} 1125 1126/** 1127 * qbman_swp_pull_mem_back() - Issue the pull dequeue command 1128 * @s: the software portal object 1129 * @d: the software portal descriptor which has been configured with 1130 * the set of qbman_pull_desc_set_*() calls 1131 * 1132 * Return 0 for success, and -EBUSY if the software portal is not ready 1133 * to do pull dequeue. 1134 */ 1135static 1136int qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d) 1137{ 1138 struct qbman_pull_desc *p; 1139 1140 if (!atomic_dec_and_test(&s->vdq.available)) { 1141 atomic_inc(&s->vdq.available); 1142 return -EBUSY; 1143 } 1144 s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt; 1145 if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) 1146 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR); 1147 else 1148 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM); 1149 p->numf = d->numf; 1150 p->tok = QMAN_DQ_TOKEN_VALID; 1151 p->dq_src = d->dq_src; 1152 p->rsp_addr = d->rsp_addr; 1153 p->rsp_addr_virt = d->rsp_addr_virt; 1154 1155 /* Set the verb byte, have to substitute in the valid-bit */ 1156 p->verb = d->verb | s->vdq.valid_bit; 1157 s->vdq.valid_bit ^= QB_VALID_BIT; 1158 dma_wmb(); 1159 qbman_write_register(s, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE); 1160 1161 return 0; 1162} 1163 1164#define QMAN_DQRR_PI_MASK 0xf 1165 1166/** 1167 * qbman_swp_dqrr_next_direct() - Get an valid DQRR entry 1168 * @s: the software portal object 1169 * 1170 * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry 1171 * only once, so repeated calls can return a sequence of DQRR entries, without 1172 * requiring they be consumed immediately or in any particular order. 1173 */ 1174const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s) 1175{ 1176 u32 verb; 1177 u32 response_verb; 1178 u32 flags; 1179 struct dpaa2_dq *p; 1180 1181 /* Before using valid-bit to detect if something is there, we have to 1182 * handle the case of the DQRR reset bug... 1183 */ 1184 if (unlikely(s->dqrr.reset_bug)) { 1185 /* 1186 * We pick up new entries by cache-inhibited producer index, 1187 * which means that a non-coherent mapping would require us to 1188 * invalidate and read *only* once that PI has indicated that 1189 * there's an entry here. The first trip around the DQRR ring 1190 * will be much less efficient than all subsequent trips around 1191 * it... 1192 */ 1193 u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) & 1194 QMAN_DQRR_PI_MASK; 1195 1196 /* there are new entries if pi != next_idx */ 1197 if (pi == s->dqrr.next_idx) 1198 return NULL; 1199 1200 /* 1201 * if next_idx is/was the last ring index, and 'pi' is 1202 * different, we can disable the workaround as all the ring 1203 * entries have now been DMA'd to so valid-bit checking is 1204 * repaired. Note: this logic needs to be based on next_idx 1205 * (which increments one at a time), rather than on pi (which 1206 * can burst and wrap-around between our snapshots of it). 1207 */ 1208 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) { 1209 pr_debug("next_idx=%d, pi=%d, clear reset bug\n", 1210 s->dqrr.next_idx, pi); 1211 s->dqrr.reset_bug = 0; 1212 } 1213 prefetch(qbman_get_cmd(s, 1214 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1215 } 1216 1217 p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); 1218 verb = p->dq.verb; 1219 1220 /* 1221 * If the valid-bit isn't of the expected polarity, nothing there. Note, 1222 * in the DQRR reset bug workaround, we shouldn't need to skip these 1223 * check, because we've already determined that a new entry is available 1224 * and we've invalidated the cacheline before reading it, so the 1225 * valid-bit behaviour is repaired and should tell us what we already 1226 * knew from reading PI. 1227 */ 1228 if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) { 1229 prefetch(qbman_get_cmd(s, 1230 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1231 return NULL; 1232 } 1233 /* 1234 * There's something there. Move "next_idx" attention to the next ring 1235 * entry (and prefetch it) before returning what we found. 1236 */ 1237 s->dqrr.next_idx++; 1238 s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */ 1239 if (!s->dqrr.next_idx) 1240 s->dqrr.valid_bit ^= QB_VALID_BIT; 1241 1242 /* 1243 * If this is the final response to a volatile dequeue command 1244 * indicate that the vdq is available 1245 */ 1246 flags = p->dq.stat; 1247 response_verb = verb & QBMAN_RESULT_MASK; 1248 if ((response_verb == QBMAN_RESULT_DQ) && 1249 (flags & DPAA2_DQ_STAT_VOLATILE) && 1250 (flags & DPAA2_DQ_STAT_EXPIRED)) 1251 atomic_inc(&s->vdq.available); 1252 1253 prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1254 1255 return p; 1256} 1257 1258/** 1259 * qbman_swp_dqrr_next_mem_back() - Get an valid DQRR entry 1260 * @s: the software portal object 1261 * 1262 * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry 1263 * only once, so repeated calls can return a sequence of DQRR entries, without 1264 * requiring they be consumed immediately or in any particular order. 1265 */ 1266const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s) 1267{ 1268 u32 verb; 1269 u32 response_verb; 1270 u32 flags; 1271 struct dpaa2_dq *p; 1272 1273 /* Before using valid-bit to detect if something is there, we have to 1274 * handle the case of the DQRR reset bug... 1275 */ 1276 if (unlikely(s->dqrr.reset_bug)) { 1277 /* 1278 * We pick up new entries by cache-inhibited producer index, 1279 * which means that a non-coherent mapping would require us to 1280 * invalidate and read *only* once that PI has indicated that 1281 * there's an entry here. The first trip around the DQRR ring 1282 * will be much less efficient than all subsequent trips around 1283 * it... 1284 */ 1285 u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) & 1286 QMAN_DQRR_PI_MASK; 1287 1288 /* there are new entries if pi != next_idx */ 1289 if (pi == s->dqrr.next_idx) 1290 return NULL; 1291 1292 /* 1293 * if next_idx is/was the last ring index, and 'pi' is 1294 * different, we can disable the workaround as all the ring 1295 * entries have now been DMA'd to so valid-bit checking is 1296 * repaired. Note: this logic needs to be based on next_idx 1297 * (which increments one at a time), rather than on pi (which 1298 * can burst and wrap-around between our snapshots of it). 1299 */ 1300 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) { 1301 pr_debug("next_idx=%d, pi=%d, clear reset bug\n", 1302 s->dqrr.next_idx, pi); 1303 s->dqrr.reset_bug = 0; 1304 } 1305 prefetch(qbman_get_cmd(s, 1306 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1307 } 1308 1309 p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx)); 1310 verb = p->dq.verb; 1311 1312 /* 1313 * If the valid-bit isn't of the expected polarity, nothing there. Note, 1314 * in the DQRR reset bug workaround, we shouldn't need to skip these 1315 * check, because we've already determined that a new entry is available 1316 * and we've invalidated the cacheline before reading it, so the 1317 * valid-bit behaviour is repaired and should tell us what we already 1318 * knew from reading PI. 1319 */ 1320 if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) { 1321 prefetch(qbman_get_cmd(s, 1322 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1323 return NULL; 1324 } 1325 /* 1326 * There's something there. Move "next_idx" attention to the next ring 1327 * entry (and prefetch it) before returning what we found. 1328 */ 1329 s->dqrr.next_idx++; 1330 s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */ 1331 if (!s->dqrr.next_idx) 1332 s->dqrr.valid_bit ^= QB_VALID_BIT; 1333 1334 /* 1335 * If this is the final response to a volatile dequeue command 1336 * indicate that the vdq is available 1337 */ 1338 flags = p->dq.stat; 1339 response_verb = verb & QBMAN_RESULT_MASK; 1340 if ((response_verb == QBMAN_RESULT_DQ) && 1341 (flags & DPAA2_DQ_STAT_VOLATILE) && 1342 (flags & DPAA2_DQ_STAT_EXPIRED)) 1343 atomic_inc(&s->vdq.available); 1344 1345 prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1346 1347 return p; 1348} 1349 1350/** 1351 * qbman_swp_dqrr_consume() - Consume DQRR entries previously returned from 1352 * qbman_swp_dqrr_next(). 1353 * @s: the software portal object 1354 * @dq: the DQRR entry to be consumed 1355 */ 1356void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct dpaa2_dq *dq) 1357{ 1358 qbman_write_register(s, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq)); 1359} 1360 1361/** 1362 * qbman_result_has_new_result() - Check and get the dequeue response from the 1363 * dq storage memory set in pull dequeue command 1364 * @s: the software portal object 1365 * @dq: the dequeue result read from the memory 1366 * 1367 * Return 1 for getting a valid dequeue result, or 0 for not getting a valid 1368 * dequeue result. 1369 * 1370 * Only used for user-provided storage of dequeue results, not DQRR. For 1371 * efficiency purposes, the driver will perform any required endianness 1372 * conversion to ensure that the user's dequeue result storage is in host-endian 1373 * format. As such, once the user has called qbman_result_has_new_result() and 1374 * been returned a valid dequeue result, they should not call it again on 1375 * the same memory location (except of course if another dequeue command has 1376 * been executed to produce a new result to that location). 1377 */ 1378int qbman_result_has_new_result(struct qbman_swp *s, const struct dpaa2_dq *dq) 1379{ 1380 if (dq->dq.tok != QMAN_DQ_TOKEN_VALID) 1381 return 0; 1382 1383 /* 1384 * Set token to be 0 so we will detect change back to 1 1385 * next time the looping is traversed. Const is cast away here 1386 * as we want users to treat the dequeue responses as read only. 1387 */ 1388 ((struct dpaa2_dq *)dq)->dq.tok = 0; 1389 1390 /* 1391 * Determine whether VDQCR is available based on whether the 1392 * current result is sitting in the first storage location of 1393 * the busy command. 1394 */ 1395 if (s->vdq.storage == dq) { 1396 s->vdq.storage = NULL; 1397 atomic_inc(&s->vdq.available); 1398 } 1399 1400 return 1; 1401} 1402 1403/** 1404 * qbman_release_desc_clear() - Clear the contents of a descriptor to 1405 * default/starting state. 1406 * @d: the pull dequeue descriptor to be cleared 1407 */ 1408void qbman_release_desc_clear(struct qbman_release_desc *d) 1409{ 1410 memset(d, 0, sizeof(*d)); 1411 d->verb = 1 << 5; /* Release Command Valid */ 1412} 1413 1414/** 1415 * qbman_release_desc_set_bpid() - Set the ID of the buffer pool to release to 1416 * @d: the pull dequeue descriptor to be set 1417 * @bpid: the bpid value to be set 1418 */ 1419void qbman_release_desc_set_bpid(struct qbman_release_desc *d, u16 bpid) 1420{ 1421 d->bpid = cpu_to_le16(bpid); 1422} 1423 1424/** 1425 * qbman_release_desc_set_rcdi() - Determines whether or not the portal's RCDI 1426 * interrupt source should be asserted after the release command is completed. 1427 * @d: the pull dequeue descriptor to be set 1428 * @enable: enable (1) or disable (0) value 1429 */ 1430void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable) 1431{ 1432 if (enable) 1433 d->verb |= 1 << 6; 1434 else 1435 d->verb &= ~(1 << 6); 1436} 1437 1438#define RAR_IDX(rar) ((rar) & 0x7) 1439#define RAR_VB(rar) ((rar) & 0x80) 1440#define RAR_SUCCESS(rar) ((rar) & 0x100) 1441 1442/** 1443 * qbman_swp_release_direct() - Issue a buffer release command 1444 * @s: the software portal object 1445 * @d: the release descriptor 1446 * @buffers: a pointer pointing to the buffer address to be released 1447 * @num_buffers: number of buffers to be released, must be less than 8 1448 * 1449 * Return 0 for success, -EBUSY if the release command ring is not ready. 1450 */ 1451int qbman_swp_release_direct(struct qbman_swp *s, 1452 const struct qbman_release_desc *d, 1453 const u64 *buffers, unsigned int num_buffers) 1454{ 1455 int i; 1456 struct qbman_release_desc *p; 1457 u32 rar; 1458 1459 if (!num_buffers || (num_buffers > 7)) 1460 return -EINVAL; 1461 1462 rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR); 1463 if (!RAR_SUCCESS(rar)) 1464 return -EBUSY; 1465 1466 /* Start the release command */ 1467 p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); 1468 1469 /* Copy the caller's buffer pointers to the command */ 1470 for (i = 0; i < num_buffers; i++) 1471 p->buf[i] = cpu_to_le64(buffers[i]); 1472 p->bpid = d->bpid; 1473 1474 /* 1475 * Set the verb byte, have to substitute in the valid-bit 1476 * and the number of buffers. 1477 */ 1478 dma_wmb(); 1479 p->verb = d->verb | RAR_VB(rar) | num_buffers; 1480 1481 return 0; 1482} 1483 1484/** 1485 * qbman_swp_release_mem_back() - Issue a buffer release command 1486 * @s: the software portal object 1487 * @d: the release descriptor 1488 * @buffers: a pointer pointing to the buffer address to be released 1489 * @num_buffers: number of buffers to be released, must be less than 8 1490 * 1491 * Return 0 for success, -EBUSY if the release command ring is not ready. 1492 */ 1493int qbman_swp_release_mem_back(struct qbman_swp *s, 1494 const struct qbman_release_desc *d, 1495 const u64 *buffers, unsigned int num_buffers) 1496{ 1497 int i; 1498 struct qbman_release_desc *p; 1499 u32 rar; 1500 1501 if (!num_buffers || (num_buffers > 7)) 1502 return -EINVAL; 1503 1504 rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR); 1505 if (!RAR_SUCCESS(rar)) 1506 return -EBUSY; 1507 1508 /* Start the release command */ 1509 p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar))); 1510 1511 /* Copy the caller's buffer pointers to the command */ 1512 for (i = 0; i < num_buffers; i++) 1513 p->buf[i] = cpu_to_le64(buffers[i]); 1514 p->bpid = d->bpid; 1515 1516 p->verb = d->verb | RAR_VB(rar) | num_buffers; 1517 dma_wmb(); 1518 qbman_write_register(s, QBMAN_CINH_SWP_RCR_AM_RT + 1519 RAR_IDX(rar) * 4, QMAN_RT_MODE); 1520 1521 return 0; 1522} 1523 1524struct qbman_acquire_desc { 1525 u8 verb; 1526 u8 reserved; 1527 __le16 bpid; 1528 u8 num; 1529 u8 reserved2[59]; 1530}; 1531 1532struct qbman_acquire_rslt { 1533 u8 verb; 1534 u8 rslt; 1535 __le16 reserved; 1536 u8 num; 1537 u8 reserved2[3]; 1538 __le64 buf[7]; 1539}; 1540 1541/** 1542 * qbman_swp_acquire() - Issue a buffer acquire command 1543 * @s: the software portal object 1544 * @bpid: the buffer pool index 1545 * @buffers: a pointer pointing to the acquired buffer addresses 1546 * @num_buffers: number of buffers to be acquired, must be less than 8 1547 * 1548 * Return 0 for success, or negative error code if the acquire command 1549 * fails. 1550 */ 1551int qbman_swp_acquire(struct qbman_swp *s, u16 bpid, u64 *buffers, 1552 unsigned int num_buffers) 1553{ 1554 struct qbman_acquire_desc *p; 1555 struct qbman_acquire_rslt *r; 1556 int i; 1557 1558 if (!num_buffers || (num_buffers > 7)) 1559 return -EINVAL; 1560 1561 /* Start the management command */ 1562 p = qbman_swp_mc_start(s); 1563 1564 if (!p) 1565 return -EBUSY; 1566 1567 /* Encode the caller-provided attributes */ 1568 p->bpid = cpu_to_le16(bpid); 1569 p->num = num_buffers; 1570 1571 /* Complete the management command */ 1572 r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE); 1573 if (unlikely(!r)) { 1574 pr_err("qbman: acquire from BPID %d failed, no response\n", 1575 bpid); 1576 return -EIO; 1577 } 1578 1579 /* Decode the outcome */ 1580 WARN_ON((r->verb & 0x7f) != QBMAN_MC_ACQUIRE); 1581 1582 /* Determine success or failure */ 1583 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { 1584 pr_err("qbman: acquire from BPID 0x%x failed, code=0x%02x\n", 1585 bpid, r->rslt); 1586 return -EIO; 1587 } 1588 1589 WARN_ON(r->num > num_buffers); 1590 1591 /* Copy the acquired buffers to the caller's array */ 1592 for (i = 0; i < r->num; i++) 1593 buffers[i] = le64_to_cpu(r->buf[i]); 1594 1595 return (int)r->num; 1596} 1597 1598struct qbman_alt_fq_state_desc { 1599 u8 verb; 1600 u8 reserved[3]; 1601 __le32 fqid; 1602 u8 reserved2[56]; 1603}; 1604 1605struct qbman_alt_fq_state_rslt { 1606 u8 verb; 1607 u8 rslt; 1608 u8 reserved[62]; 1609}; 1610 1611#define ALT_FQ_FQID_MASK 0x00FFFFFF 1612 1613int qbman_swp_alt_fq_state(struct qbman_swp *s, u32 fqid, 1614 u8 alt_fq_verb) 1615{ 1616 struct qbman_alt_fq_state_desc *p; 1617 struct qbman_alt_fq_state_rslt *r; 1618 1619 /* Start the management command */ 1620 p = qbman_swp_mc_start(s); 1621 if (!p) 1622 return -EBUSY; 1623 1624 p->fqid = cpu_to_le32(fqid & ALT_FQ_FQID_MASK); 1625 1626 /* Complete the management command */ 1627 r = qbman_swp_mc_complete(s, p, alt_fq_verb); 1628 if (unlikely(!r)) { 1629 pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n", 1630 alt_fq_verb); 1631 return -EIO; 1632 } 1633 1634 /* Decode the outcome */ 1635 WARN_ON((r->verb & QBMAN_RESULT_MASK) != alt_fq_verb); 1636 1637 /* Determine success or failure */ 1638 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { 1639 pr_err("qbman: ALT FQID %d failed: verb = 0x%08x code = 0x%02x\n", 1640 fqid, r->verb, r->rslt); 1641 return -EIO; 1642 } 1643 1644 return 0; 1645} 1646 1647struct qbman_cdan_ctrl_desc { 1648 u8 verb; 1649 u8 reserved; 1650 __le16 ch; 1651 u8 we; 1652 u8 ctrl; 1653 __le16 reserved2; 1654 __le64 cdan_ctx; 1655 u8 reserved3[48]; 1656 1657}; 1658 1659struct qbman_cdan_ctrl_rslt { 1660 u8 verb; 1661 u8 rslt; 1662 __le16 ch; 1663 u8 reserved[60]; 1664}; 1665 1666int qbman_swp_CDAN_set(struct qbman_swp *s, u16 channelid, 1667 u8 we_mask, u8 cdan_en, 1668 u64 ctx) 1669{ 1670 struct qbman_cdan_ctrl_desc *p = NULL; 1671 struct qbman_cdan_ctrl_rslt *r = NULL; 1672 1673 /* Start the management command */ 1674 p = qbman_swp_mc_start(s); 1675 if (!p) 1676 return -EBUSY; 1677 1678 /* Encode the caller-provided attributes */ 1679 p->ch = cpu_to_le16(channelid); 1680 p->we = we_mask; 1681 if (cdan_en) 1682 p->ctrl = 1; 1683 else 1684 p->ctrl = 0; 1685 p->cdan_ctx = cpu_to_le64(ctx); 1686 1687 /* Complete the management command */ 1688 r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE); 1689 if (unlikely(!r)) { 1690 pr_err("qbman: wqchan config failed, no response\n"); 1691 return -EIO; 1692 } 1693 1694 WARN_ON((r->verb & 0x7f) != QBMAN_WQCHAN_CONFIGURE); 1695 1696 /* Determine success or failure */ 1697 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { 1698 pr_err("qbman: CDAN cQID %d failed: code = 0x%02x\n", 1699 channelid, r->rslt); 1700 return -EIO; 1701 } 1702 1703 return 0; 1704} 1705 1706#define QBMAN_RESPONSE_VERB_MASK 0x7f 1707#define QBMAN_FQ_QUERY_NP 0x45 1708#define QBMAN_BP_QUERY 0x32 1709 1710struct qbman_fq_query_desc { 1711 u8 verb; 1712 u8 reserved[3]; 1713 __le32 fqid; 1714 u8 reserved2[56]; 1715}; 1716 1717int qbman_fq_query_state(struct qbman_swp *s, u32 fqid, 1718 struct qbman_fq_query_np_rslt *r) 1719{ 1720 struct qbman_fq_query_desc *p; 1721 void *resp; 1722 1723 p = (struct qbman_fq_query_desc *)qbman_swp_mc_start(s); 1724 if (!p) 1725 return -EBUSY; 1726 1727 /* FQID is a 24 bit value */ 1728 p->fqid = cpu_to_le32(fqid & 0x00FFFFFF); 1729 resp = qbman_swp_mc_complete(s, p, QBMAN_FQ_QUERY_NP); 1730 if (!resp) { 1731 pr_err("qbman: Query FQID %d NP fields failed, no response\n", 1732 fqid); 1733 return -EIO; 1734 } 1735 *r = *(struct qbman_fq_query_np_rslt *)resp; 1736 /* Decode the outcome */ 1737 WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_FQ_QUERY_NP); 1738 1739 /* Determine success or failure */ 1740 if (r->rslt != QBMAN_MC_RSLT_OK) { 1741 pr_err("Query NP fields of FQID 0x%x failed, code=0x%02x\n", 1742 p->fqid, r->rslt); 1743 return -EIO; 1744 } 1745 1746 return 0; 1747} 1748 1749u32 qbman_fq_state_frame_count(const struct qbman_fq_query_np_rslt *r) 1750{ 1751 return (le32_to_cpu(r->frm_cnt) & 0x00FFFFFF); 1752} 1753 1754u32 qbman_fq_state_byte_count(const struct qbman_fq_query_np_rslt *r) 1755{ 1756 return le32_to_cpu(r->byte_cnt); 1757} 1758 1759struct qbman_bp_query_desc { 1760 u8 verb; 1761 u8 reserved; 1762 __le16 bpid; 1763 u8 reserved2[60]; 1764}; 1765 1766int qbman_bp_query(struct qbman_swp *s, u16 bpid, 1767 struct qbman_bp_query_rslt *r) 1768{ 1769 struct qbman_bp_query_desc *p; 1770 void *resp; 1771 1772 p = (struct qbman_bp_query_desc *)qbman_swp_mc_start(s); 1773 if (!p) 1774 return -EBUSY; 1775 1776 p->bpid = cpu_to_le16(bpid); 1777 resp = qbman_swp_mc_complete(s, p, QBMAN_BP_QUERY); 1778 if (!resp) { 1779 pr_err("qbman: Query BPID %d fields failed, no response\n", 1780 bpid); 1781 return -EIO; 1782 } 1783 *r = *(struct qbman_bp_query_rslt *)resp; 1784 /* Decode the outcome */ 1785 WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_BP_QUERY); 1786 1787 /* Determine success or failure */ 1788 if (r->rslt != QBMAN_MC_RSLT_OK) { 1789 pr_err("Query fields of BPID 0x%x failed, code=0x%02x\n", 1790 bpid, r->rslt); 1791 return -EIO; 1792 } 1793 1794 return 0; 1795} 1796 1797u32 qbman_bp_info_num_free_bufs(struct qbman_bp_query_rslt *a) 1798{ 1799 return le32_to_cpu(a->fill); 1800} 1801 1802/** 1803 * qbman_swp_set_irq_coalescing() - Set new IRQ coalescing values 1804 * @p: the software portal object 1805 * @irq_threshold: interrupt threshold 1806 * @irq_holdoff: interrupt holdoff (timeout) period in us 1807 * 1808 * Return 0 for success, or negative error code on error. 1809 */ 1810int qbman_swp_set_irq_coalescing(struct qbman_swp *p, u32 irq_threshold, 1811 u32 irq_holdoff) 1812{ 1813 u32 itp, max_holdoff; 1814 1815 /* Convert irq_holdoff value from usecs to 256 QBMAN clock cycles 1816 * increments. This depends on the QBMAN internal frequency. 1817 */ 1818 itp = (irq_holdoff * 1000) / p->desc->qman_256_cycles_per_ns; 1819 if (itp > 4096) { 1820 max_holdoff = (p->desc->qman_256_cycles_per_ns * 4096) / 1000; 1821 pr_err("irq_holdoff must be <= %uus\n", max_holdoff); 1822 return -EINVAL; 1823 } 1824 1825 if (irq_threshold >= p->dqrr.dqrr_size) { 1826 pr_err("irq_threshold must be < %u\n", p->dqrr.dqrr_size - 1); 1827 return -EINVAL; 1828 } 1829 1830 p->irq_threshold = irq_threshold; 1831 p->irq_holdoff = irq_holdoff; 1832 1833 qbman_write_register(p, QBMAN_CINH_SWP_DQRR_ITR, irq_threshold); 1834 qbman_write_register(p, QBMAN_CINH_SWP_ITPR, itp); 1835 1836 return 0; 1837} 1838 1839/** 1840 * qbman_swp_get_irq_coalescing() - Get the current IRQ coalescing parameters 1841 * @p: the software portal object 1842 * @irq_threshold: interrupt threshold (an IRQ is generated when there are more 1843 * DQRR entries in the portal than the threshold) 1844 * @irq_holdoff: interrupt holdoff (timeout) period in us 1845 */ 1846void qbman_swp_get_irq_coalescing(struct qbman_swp *p, u32 *irq_threshold, 1847 u32 *irq_holdoff) 1848{ 1849 if (irq_threshold) 1850 *irq_threshold = p->irq_threshold; 1851 if (irq_holdoff) 1852 *irq_holdoff = p->irq_holdoff; 1853} 1854