1/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ 2/* $NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ 3 4/* 5 * Copyright (c) 2019 Internet Initiative Japan, Inc. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30/* 31 * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 37 * * Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * * Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in 41 * the documentation and/or other materials provided with the 42 * distribution. 43 * * Neither the name of Intel Corporation nor the names of its 44 * contributors may be used to endorse or promote products derived 45 * from this software without specific prior written permission. 46 * 47 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 48 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 49 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 50 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 51 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 52 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 53 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 54 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 55 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 56 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 57 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 58 */ 59 60#include <sys/cdefs.h> 61__FBSDID("$FreeBSD$"); 62#if 0 63__KERNEL_RCSID(0, "$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); 64#endif 65 66#include <sys/param.h> 67#include <sys/bus.h> 68#include <sys/firmware.h> 69#include <sys/limits.h> 70#include <sys/systm.h> 71 72#include <machine/bus.h> 73 74#include <dev/pci/pcireg.h> 75#include <dev/pci/pcivar.h> 76 77#include "qatreg.h" 78#include "qatvar.h" 79#include "qat_aevar.h" 80 81static int qat_ae_write_4(struct qat_softc *, u_char, bus_size_t, 82 uint32_t); 83static int qat_ae_read_4(struct qat_softc *, u_char, bus_size_t, 84 uint32_t *); 85static void qat_ae_ctx_indr_write(struct qat_softc *, u_char, uint32_t, 86 bus_size_t, uint32_t); 87static int qat_ae_ctx_indr_read(struct qat_softc *, u_char, uint32_t, 88 bus_size_t, uint32_t *); 89 90static u_short qat_aereg_get_10bit_addr(enum aereg_type, u_short); 91static int qat_aereg_rel_data_write(struct qat_softc *, u_char, u_char, 92 enum aereg_type, u_short, uint32_t); 93static int qat_aereg_rel_data_read(struct qat_softc *, u_char, u_char, 94 enum aereg_type, u_short, uint32_t *); 95static int qat_aereg_rel_rdxfer_write(struct qat_softc *, u_char, u_char, 96 enum aereg_type, u_short, uint32_t); 97static int qat_aereg_rel_wrxfer_write(struct qat_softc *, u_char, u_char, 98 enum aereg_type, u_short, uint32_t); 99static int qat_aereg_rel_nn_write(struct qat_softc *, u_char, u_char, 100 enum aereg_type, u_short, uint32_t); 101static int qat_aereg_abs_to_rel(struct qat_softc *, u_char, u_short, 102 u_short *, u_char *); 103static int qat_aereg_abs_data_write(struct qat_softc *, u_char, 104 enum aereg_type, u_short, uint32_t); 105 106static void qat_ae_enable_ctx(struct qat_softc *, u_char, u_int); 107static void qat_ae_disable_ctx(struct qat_softc *, u_char, u_int); 108static void qat_ae_write_ctx_mode(struct qat_softc *, u_char, u_char); 109static void qat_ae_write_nn_mode(struct qat_softc *, u_char, u_char); 110static void qat_ae_write_lm_mode(struct qat_softc *, u_char, 111 enum aereg_type, u_char); 112static void qat_ae_write_shared_cs_mode0(struct qat_softc *, u_char, 113 u_char); 114static void qat_ae_write_shared_cs_mode(struct qat_softc *, u_char, u_char); 115static int qat_ae_set_reload_ustore(struct qat_softc *, u_char, u_int, int, 116 u_int); 117 118static enum qat_ae_status qat_ae_get_status(struct qat_softc *, u_char); 119static int qat_ae_is_active(struct qat_softc *, u_char); 120static int qat_ae_wait_num_cycles(struct qat_softc *, u_char, int, int); 121 122static int qat_ae_clear_reset(struct qat_softc *); 123static int qat_ae_check(struct qat_softc *); 124static int qat_ae_reset_timestamp(struct qat_softc *); 125static void qat_ae_clear_xfer(struct qat_softc *); 126static int qat_ae_clear_gprs(struct qat_softc *); 127 128static void qat_ae_get_shared_ustore_ae(u_char, u_char *); 129static u_int qat_ae_ucode_parity64(uint64_t); 130static uint64_t qat_ae_ucode_set_ecc(uint64_t); 131static int qat_ae_ucode_write(struct qat_softc *, u_char, u_int, u_int, 132 const uint64_t *); 133static int qat_ae_ucode_read(struct qat_softc *, u_char, u_int, u_int, 134 uint64_t *); 135static u_int qat_ae_concat_ucode(uint64_t *, u_int, u_int, u_int, u_int *); 136static int qat_ae_exec_ucode(struct qat_softc *, u_char, u_char, 137 uint64_t *, u_int, int, u_int, u_int *); 138static int qat_ae_exec_ucode_init_lm(struct qat_softc *, u_char, u_char, 139 int *, uint64_t *, u_int, 140 u_int *, u_int *, u_int *, u_int *, u_int *); 141static int qat_ae_restore_init_lm_gprs(struct qat_softc *, u_char, u_char, 142 u_int, u_int, u_int, u_int, u_int); 143static int qat_ae_get_inst_num(int); 144static int qat_ae_batch_put_lm(struct qat_softc *, u_char, 145 struct qat_ae_batch_init_list *, size_t); 146static int qat_ae_write_pc(struct qat_softc *, u_char, u_int, u_int); 147 148static u_int qat_aefw_csum(char *, int); 149static const char *qat_aefw_uof_string(struct qat_softc *, size_t); 150static struct uof_chunk_hdr *qat_aefw_uof_find_chunk(struct qat_softc *, 151 const char *, struct uof_chunk_hdr *); 152 153static int qat_aefw_load_mof(struct qat_softc *); 154static void qat_aefw_unload_mof(struct qat_softc *); 155static int qat_aefw_load_mmp(struct qat_softc *); 156static void qat_aefw_unload_mmp(struct qat_softc *); 157 158static int qat_aefw_mof_find_uof0(struct qat_softc *, 159 struct mof_uof_hdr *, struct mof_uof_chunk_hdr *, 160 u_int, size_t, const char *, 161 size_t *, void **); 162static int qat_aefw_mof_find_uof(struct qat_softc *); 163static int qat_aefw_mof_parse(struct qat_softc *); 164 165static int qat_aefw_uof_parse_image(struct qat_softc *, 166 struct qat_uof_image *, struct uof_chunk_hdr *uch); 167static int qat_aefw_uof_parse_images(struct qat_softc *); 168static int qat_aefw_uof_parse(struct qat_softc *); 169 170static int qat_aefw_alloc_auth_dmamem(struct qat_softc *, char *, size_t, 171 struct qat_dmamem *); 172static int qat_aefw_auth(struct qat_softc *, struct qat_dmamem *); 173static int qat_aefw_suof_load(struct qat_softc *sc, 174 struct qat_dmamem *dma); 175static int qat_aefw_suof_parse_image(struct qat_softc *, 176 struct qat_suof_image *, struct suof_chunk_hdr *); 177static int qat_aefw_suof_parse(struct qat_softc *); 178static int qat_aefw_suof_write(struct qat_softc *); 179 180static int qat_aefw_uof_assign_image(struct qat_softc *, struct qat_ae *, 181 struct qat_uof_image *); 182static int qat_aefw_uof_init_ae(struct qat_softc *, u_char); 183static int qat_aefw_uof_init(struct qat_softc *); 184 185static int qat_aefw_init_memory_one(struct qat_softc *, 186 struct uof_init_mem *); 187static void qat_aefw_free_lm_init(struct qat_softc *, u_char); 188static int qat_aefw_init_ustore(struct qat_softc *); 189static int qat_aefw_init_reg(struct qat_softc *, u_char, u_char, 190 enum aereg_type, u_short, u_int); 191static int qat_aefw_init_reg_sym_expr(struct qat_softc *, u_char, 192 struct qat_uof_image *); 193static int qat_aefw_init_memory(struct qat_softc *); 194static int qat_aefw_init_globals(struct qat_softc *); 195static uint64_t qat_aefw_get_uof_inst(struct qat_softc *, 196 struct qat_uof_page *, u_int); 197static int qat_aefw_do_pagein(struct qat_softc *, u_char, 198 struct qat_uof_page *); 199static int qat_aefw_uof_write_one(struct qat_softc *, 200 struct qat_uof_image *); 201static int qat_aefw_uof_write(struct qat_softc *); 202 203static int 204qat_ae_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset, 205 uint32_t value) 206{ 207 int times = TIMEOUT_AE_CSR; 208 209 do { 210 qat_ae_local_write_4(sc, ae, offset, value); 211 if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) & 212 LOCAL_CSR_STATUS_STATUS) == 0) 213 return 0; 214 215 } while (times--); 216 217 device_printf(sc->sc_dev, 218 "couldn't write AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset); 219 return EFAULT; 220} 221 222static int 223qat_ae_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset, 224 uint32_t *value) 225{ 226 int times = TIMEOUT_AE_CSR; 227 uint32_t v; 228 229 do { 230 v = qat_ae_local_read_4(sc, ae, offset); 231 if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) & 232 LOCAL_CSR_STATUS_STATUS) == 0) { 233 *value = v; 234 return 0; 235 } 236 } while (times--); 237 238 device_printf(sc->sc_dev, 239 "couldn't read AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset); 240 return EFAULT; 241} 242 243static void 244qat_ae_ctx_indr_write(struct qat_softc *sc, u_char ae, uint32_t ctx_mask, 245 bus_size_t offset, uint32_t value) 246{ 247 int ctx; 248 uint32_t ctxptr; 249 250 MPASS(offset == CTX_FUTURE_COUNT_INDIRECT || 251 offset == FUTURE_COUNT_SIGNAL_INDIRECT || 252 offset == CTX_STS_INDIRECT || 253 offset == CTX_WAKEUP_EVENTS_INDIRECT || 254 offset == CTX_SIG_EVENTS_INDIRECT || 255 offset == LM_ADDR_0_INDIRECT || 256 offset == LM_ADDR_1_INDIRECT || 257 offset == INDIRECT_LM_ADDR_0_BYTE_INDEX || 258 offset == INDIRECT_LM_ADDR_1_BYTE_INDEX); 259 260 qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr); 261 for (ctx = 0; ctx < MAX_AE_CTX; ctx++) { 262 if ((ctx_mask & (1 << ctx)) == 0) 263 continue; 264 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx); 265 qat_ae_write_4(sc, ae, offset, value); 266 } 267 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr); 268} 269 270static int 271qat_ae_ctx_indr_read(struct qat_softc *sc, u_char ae, uint32_t ctx, 272 bus_size_t offset, uint32_t *value) 273{ 274 int error; 275 uint32_t ctxptr; 276 277 MPASS(offset == CTX_FUTURE_COUNT_INDIRECT || 278 offset == FUTURE_COUNT_SIGNAL_INDIRECT || 279 offset == CTX_STS_INDIRECT || 280 offset == CTX_WAKEUP_EVENTS_INDIRECT || 281 offset == CTX_SIG_EVENTS_INDIRECT || 282 offset == LM_ADDR_0_INDIRECT || 283 offset == LM_ADDR_1_INDIRECT || 284 offset == INDIRECT_LM_ADDR_0_BYTE_INDEX || 285 offset == INDIRECT_LM_ADDR_1_BYTE_INDEX); 286 287 /* save the ctx ptr */ 288 qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr); 289 if ((ctxptr & CSR_CTX_POINTER_CONTEXT) != 290 (ctx & CSR_CTX_POINTER_CONTEXT)) 291 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx); 292 293 error = qat_ae_read_4(sc, ae, offset, value); 294 295 /* restore ctx ptr */ 296 if ((ctxptr & CSR_CTX_POINTER_CONTEXT) != 297 (ctx & CSR_CTX_POINTER_CONTEXT)) 298 qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr); 299 300 return error; 301} 302 303static u_short 304qat_aereg_get_10bit_addr(enum aereg_type regtype, u_short reg) 305{ 306 u_short addr; 307 308 switch (regtype) { 309 case AEREG_GPA_ABS: 310 case AEREG_GPB_ABS: 311 addr = (reg & 0x7f) | 0x80; 312 break; 313 case AEREG_GPA_REL: 314 case AEREG_GPB_REL: 315 addr = reg & 0x1f; 316 break; 317 case AEREG_SR_RD_REL: 318 case AEREG_SR_WR_REL: 319 case AEREG_SR_REL: 320 addr = 0x180 | (reg & 0x1f); 321 break; 322 case AEREG_SR_INDX: 323 addr = 0x140 | ((reg & 0x3) << 1); 324 break; 325 case AEREG_DR_RD_REL: 326 case AEREG_DR_WR_REL: 327 case AEREG_DR_REL: 328 addr = 0x1c0 | (reg & 0x1f); 329 break; 330 case AEREG_DR_INDX: 331 addr = 0x100 | ((reg & 0x3) << 1); 332 break; 333 case AEREG_NEIGH_INDX: 334 addr = 0x241 | ((reg & 0x3) << 1); 335 break; 336 case AEREG_NEIGH_REL: 337 addr = 0x280 | (reg & 0x1f); 338 break; 339 case AEREG_LMEM0: 340 addr = 0x200; 341 break; 342 case AEREG_LMEM1: 343 addr = 0x220; 344 break; 345 case AEREG_NO_DEST: 346 addr = 0x300 | (reg & 0xff); 347 break; 348 default: 349 addr = AEREG_BAD_REGADDR; 350 break; 351 } 352 return (addr); 353} 354 355static int 356qat_aereg_rel_data_write(struct qat_softc *sc, u_char ae, u_char ctx, 357 enum aereg_type regtype, u_short relreg, uint32_t value) 358{ 359 uint16_t srchi, srclo, destaddr, data16hi, data16lo; 360 uint64_t inst[] = { 361 0x0F440000000ull, /* immed_w1[reg, val_hi16] */ 362 0x0F040000000ull, /* immed_w0[reg, val_lo16] */ 363 0x0F0000C0300ull, /* nop */ 364 0x0E000010000ull /* ctx_arb[kill] */ 365 }; 366 const int ninst = nitems(inst); 367 const int imm_w1 = 0, imm_w0 = 1; 368 unsigned int ctxen; 369 uint16_t mask; 370 371 /* This logic only works for GPRs and LM index registers, 372 not NN or XFER registers! */ 373 MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL || 374 regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1); 375 376 if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL)) { 377 /* determine the context mode */ 378 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 379 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 380 /* 4-ctx mode */ 381 if (ctx & 0x1) 382 return EINVAL; 383 mask = 0x1f; 384 } else { 385 /* 8-ctx mode */ 386 mask = 0x0f; 387 } 388 if (relreg & ~mask) 389 return EINVAL; 390 } 391 if ((destaddr = qat_aereg_get_10bit_addr(regtype, relreg)) == 392 AEREG_BAD_REGADDR) { 393 return EINVAL; 394 } 395 396 data16lo = 0xffff & value; 397 data16hi = 0xffff & (value >> 16); 398 srchi = qat_aereg_get_10bit_addr(AEREG_NO_DEST, 399 (uint16_t)(0xff & data16hi)); 400 srclo = qat_aereg_get_10bit_addr(AEREG_NO_DEST, 401 (uint16_t)(0xff & data16lo)); 402 403 switch (regtype) { 404 case AEREG_GPA_REL: /* A rel source */ 405 inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) | 406 ((srchi & 0x3ff) << 10) | (destaddr & 0x3ff); 407 inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) | 408 ((srclo & 0x3ff) << 10) | (destaddr & 0x3ff); 409 break; 410 default: 411 inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) | 412 ((destaddr & 0x3ff) << 10) | (srchi & 0x3ff); 413 inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) | 414 ((destaddr & 0x3ff) << 10) | (srclo & 0x3ff); 415 break; 416 } 417 418 return qat_ae_exec_ucode(sc, ae, ctx, inst, ninst, 1, ninst * 5, NULL); 419} 420 421static int 422qat_aereg_rel_data_read(struct qat_softc *sc, u_char ae, u_char ctx, 423 enum aereg_type regtype, u_short relreg, uint32_t *value) 424{ 425 uint64_t inst, savucode; 426 uint32_t ctxen, misc, nmisc, savctx, ctxarbctl, ulo, uhi; 427 u_int uaddr, ustore_addr; 428 int error; 429 u_short mask, regaddr; 430 u_char nae; 431 432 MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL || 433 regtype == AEREG_SR_REL || regtype == AEREG_SR_RD_REL || 434 regtype == AEREG_DR_REL || regtype == AEREG_DR_RD_REL || 435 regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1); 436 437 if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL) || 438 (regtype == AEREG_SR_REL) || (regtype == AEREG_SR_RD_REL) || 439 (regtype == AEREG_DR_REL) || (regtype == AEREG_DR_RD_REL)) 440 { 441 /* determine the context mode */ 442 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 443 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 444 /* 4-ctx mode */ 445 if (ctx & 0x1) 446 return EINVAL; 447 mask = 0x1f; 448 } else { 449 /* 8-ctx mode */ 450 mask = 0x0f; 451 } 452 if (relreg & ~mask) 453 return EINVAL; 454 } 455 if ((regaddr = qat_aereg_get_10bit_addr(regtype, relreg)) == 456 AEREG_BAD_REGADDR) { 457 return EINVAL; 458 } 459 460 /* instruction -- alu[--, --, B, reg] */ 461 switch (regtype) { 462 case AEREG_GPA_REL: 463 /* A rel source */ 464 inst = 0xA070000000ull | (regaddr & 0x3ff); 465 break; 466 default: 467 inst = (0xA030000000ull | ((regaddr & 0x3ff) << 10)); 468 break; 469 } 470 471 /* backup shared control store bit, and force AE to 472 * none-shared mode before executing ucode snippet */ 473 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 474 if (misc & AE_MISC_CONTROL_SHARE_CS) { 475 qat_ae_get_shared_ustore_ae(ae, &nae); 476 if ((1 << nae) & sc->sc_ae_mask && qat_ae_is_active(sc, nae)) 477 return EBUSY; 478 } 479 480 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; 481 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); 482 483 /* read current context */ 484 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx); 485 qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl); 486 487 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 488 /* prevent clearing the W1C bits: the breakpoint bit, 489 ECC error bit, and Parity error bit */ 490 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 491 492 /* change the context */ 493 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) 494 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 495 ctx & ACTIVE_CTX_STATUS_ACNO); 496 /* save a ustore location */ 497 if ((error = qat_ae_ucode_read(sc, ae, 0, 1, &savucode)) != 0) { 498 /* restore AE_MISC_CONTROL csr */ 499 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 500 501 /* restore the context */ 502 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) { 503 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 504 savctx & ACTIVE_CTX_STATUS_ACNO); 505 } 506 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); 507 508 return (error); 509 } 510 511 /* turn off ustore parity */ 512 qat_ae_write_4(sc, ae, CTX_ENABLES, 513 ctxen & (~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE)); 514 515 /* save ustore-addr csr */ 516 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); 517 518 /* write the ALU instruction to ustore, enable ecs bit */ 519 uaddr = 0 | USTORE_ADDRESS_ECS; 520 521 /* set the uaddress */ 522 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 523 inst = qat_ae_ucode_set_ecc(inst); 524 525 ulo = (uint32_t)(inst & 0xffffffff); 526 uhi = (uint32_t)(inst >> 32); 527 528 qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo); 529 530 /* this will auto increment the address */ 531 qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi); 532 533 /* set the uaddress */ 534 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 535 536 /* delay for at least 8 cycles */ 537 qat_ae_wait_num_cycles(sc, ae, 0x8, 0); 538 539 /* read ALU output -- the instruction should have been executed 540 prior to clearing the ECS in putUwords */ 541 qat_ae_read_4(sc, ae, ALU_OUT, value); 542 543 /* restore ustore-addr csr */ 544 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); 545 546 /* restore the ustore */ 547 error = qat_ae_ucode_write(sc, ae, 0, 1, &savucode); 548 549 /* restore the context */ 550 if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) { 551 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 552 savctx & ACTIVE_CTX_STATUS_ACNO); 553 } 554 555 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); 556 557 /* restore AE_MISC_CONTROL csr */ 558 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 559 560 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 561 562 return error; 563} 564 565static int 566qat_aereg_rel_rdxfer_write(struct qat_softc *sc, u_char ae, u_char ctx, 567 enum aereg_type regtype, u_short relreg, uint32_t value) 568{ 569 bus_size_t addr; 570 int error; 571 uint32_t ctxen; 572 u_short mask; 573 u_short dr_offset; 574 575 MPASS(regtype == AEREG_SR_REL || regtype == AEREG_DR_REL || 576 regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_RD_REL); 577 578 error = qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 579 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 580 if (ctx & 0x1) { 581 device_printf(sc->sc_dev, 582 "bad ctx argument in 4-ctx mode,ctx=0x%x\n", ctx); 583 return EINVAL; 584 } 585 mask = 0x1f; 586 dr_offset = 0x20; 587 588 } else { 589 mask = 0x0f; 590 dr_offset = 0x10; 591 } 592 593 if (relreg & ~mask) 594 return EINVAL; 595 596 addr = relreg + (ctx << 0x5); 597 598 switch (regtype) { 599 case AEREG_SR_REL: 600 case AEREG_SR_RD_REL: 601 qat_ae_xfer_write_4(sc, ae, addr, value); 602 break; 603 case AEREG_DR_REL: 604 case AEREG_DR_RD_REL: 605 qat_ae_xfer_write_4(sc, ae, addr + dr_offset, value); 606 break; 607 default: 608 error = EINVAL; 609 } 610 611 return error; 612} 613 614static int 615qat_aereg_rel_wrxfer_write(struct qat_softc *sc, u_char ae, u_char ctx, 616 enum aereg_type regtype, u_short relreg, uint32_t value) 617{ 618 619 panic("notyet"); 620 621 return 0; 622} 623 624static int 625qat_aereg_rel_nn_write(struct qat_softc *sc, u_char ae, u_char ctx, 626 enum aereg_type regtype, u_short relreg, uint32_t value) 627{ 628 629 panic("notyet"); 630 631 return 0; 632} 633 634static int 635qat_aereg_abs_to_rel(struct qat_softc *sc, u_char ae, 636 u_short absreg, u_short *relreg, u_char *ctx) 637{ 638 uint32_t ctxen; 639 640 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 641 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 642 /* 4-ctx mode */ 643 *relreg = absreg & 0x1f; 644 *ctx = (absreg >> 0x4) & 0x6; 645 } else { 646 /* 8-ctx mode */ 647 *relreg = absreg & 0x0f; 648 *ctx = (absreg >> 0x4) & 0x7; 649 } 650 651 return 0; 652} 653 654static int 655qat_aereg_abs_data_write(struct qat_softc *sc, u_char ae, 656 enum aereg_type regtype, u_short absreg, uint32_t value) 657{ 658 int error; 659 u_short relreg; 660 u_char ctx; 661 662 qat_aereg_abs_to_rel(sc, ae, absreg, &relreg, &ctx); 663 664 switch (regtype) { 665 case AEREG_GPA_ABS: 666 MPASS(absreg < MAX_GPR_REG); 667 error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 668 relreg, value); 669 break; 670 case AEREG_GPB_ABS: 671 MPASS(absreg < MAX_GPR_REG); 672 error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 673 relreg, value); 674 break; 675 case AEREG_DR_RD_ABS: 676 MPASS(absreg < MAX_XFER_REG); 677 error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_DR_RD_REL, 678 relreg, value); 679 break; 680 case AEREG_SR_RD_ABS: 681 MPASS(absreg < MAX_XFER_REG); 682 error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_SR_RD_REL, 683 relreg, value); 684 break; 685 case AEREG_DR_WR_ABS: 686 MPASS(absreg < MAX_XFER_REG); 687 error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_DR_WR_REL, 688 relreg, value); 689 break; 690 case AEREG_SR_WR_ABS: 691 MPASS(absreg < MAX_XFER_REG); 692 error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_SR_WR_REL, 693 relreg, value); 694 break; 695 case AEREG_NEIGH_ABS: 696 MPASS(absreg < MAX_NN_REG); 697 if (absreg >= MAX_NN_REG) 698 return EINVAL; 699 error = qat_aereg_rel_nn_write(sc, ae, ctx, AEREG_NEIGH_REL, 700 relreg, value); 701 break; 702 default: 703 panic("Invalid Register Type"); 704 } 705 706 return error; 707} 708 709static void 710qat_ae_enable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask) 711{ 712 uint32_t ctxen; 713 714 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 715 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 716 717 if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { 718 ctx_mask &= 0x55; 719 } else { 720 ctx_mask &= 0xff; 721 } 722 723 ctxen |= __SHIFTIN(ctx_mask, CTX_ENABLES_ENABLE); 724 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 725} 726 727static void 728qat_ae_disable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask) 729{ 730 uint32_t ctxen; 731 732 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 733 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 734 ctxen &= ~(__SHIFTIN(ctx_mask & AE_ALL_CTX, CTX_ENABLES_ENABLE)); 735 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 736} 737 738static void 739qat_ae_write_ctx_mode(struct qat_softc *sc, u_char ae, u_char mode) 740{ 741 uint32_t val, nval; 742 743 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 744 val &= CTX_ENABLES_IGNORE_W1C_MASK; 745 746 if (mode == 4) 747 nval = val | CTX_ENABLES_INUSE_CONTEXTS; 748 else 749 nval = val & ~CTX_ENABLES_INUSE_CONTEXTS; 750 751 if (val != nval) 752 qat_ae_write_4(sc, ae, CTX_ENABLES, nval); 753} 754 755static void 756qat_ae_write_nn_mode(struct qat_softc *sc, u_char ae, u_char mode) 757{ 758 uint32_t val, nval; 759 760 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 761 val &= CTX_ENABLES_IGNORE_W1C_MASK; 762 763 if (mode) 764 nval = val | CTX_ENABLES_NN_MODE; 765 else 766 nval = val & ~CTX_ENABLES_NN_MODE; 767 768 if (val != nval) 769 qat_ae_write_4(sc, ae, CTX_ENABLES, nval); 770} 771 772static void 773qat_ae_write_lm_mode(struct qat_softc *sc, u_char ae, 774 enum aereg_type lm, u_char mode) 775{ 776 uint32_t val, nval; 777 uint32_t bit; 778 779 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 780 val &= CTX_ENABLES_IGNORE_W1C_MASK; 781 782 switch (lm) { 783 case AEREG_LMEM0: 784 bit = CTX_ENABLES_LMADDR_0_GLOBAL; 785 break; 786 case AEREG_LMEM1: 787 bit = CTX_ENABLES_LMADDR_1_GLOBAL; 788 break; 789 default: 790 panic("invalid lmem reg type"); 791 break; 792 } 793 794 if (mode) 795 nval = val | bit; 796 else 797 nval = val & ~bit; 798 799 if (val != nval) 800 qat_ae_write_4(sc, ae, CTX_ENABLES, nval); 801} 802 803static void 804qat_ae_write_shared_cs_mode0(struct qat_softc *sc, u_char ae, u_char mode) 805{ 806 uint32_t val, nval; 807 808 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); 809 810 if (mode == 1) 811 nval = val | AE_MISC_CONTROL_SHARE_CS; 812 else 813 nval = val & ~AE_MISC_CONTROL_SHARE_CS; 814 815 if (val != nval) 816 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nval); 817} 818 819static void 820qat_ae_write_shared_cs_mode(struct qat_softc *sc, u_char ae, u_char mode) 821{ 822 u_char nae; 823 824 qat_ae_get_shared_ustore_ae(ae, &nae); 825 826 qat_ae_write_shared_cs_mode0(sc, ae, mode); 827 828 if ((sc->sc_ae_mask & (1 << nae))) { 829 qat_ae_write_shared_cs_mode0(sc, nae, mode); 830 } 831} 832 833static int 834qat_ae_set_reload_ustore(struct qat_softc *sc, u_char ae, 835 u_int reload_size, int shared_mode, u_int ustore_dram_addr) 836{ 837 uint32_t val, cs_reload; 838 839 switch (reload_size) { 840 case 0: 841 cs_reload = 0x0; 842 break; 843 case QAT_2K: 844 cs_reload = 0x1; 845 break; 846 case QAT_4K: 847 cs_reload = 0x2; 848 break; 849 case QAT_8K: 850 cs_reload = 0x3; 851 break; 852 default: 853 return EINVAL; 854 } 855 856 if (cs_reload) 857 QAT_AE(sc, ae).qae_ustore_dram_addr = ustore_dram_addr; 858 859 QAT_AE(sc, ae).qae_reload_size = reload_size; 860 861 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); 862 val &= ~(AE_MISC_CONTROL_ONE_CTX_RELOAD | 863 AE_MISC_CONTROL_CS_RELOAD | AE_MISC_CONTROL_SHARE_CS); 864 val |= __SHIFTIN(cs_reload, AE_MISC_CONTROL_CS_RELOAD) | 865 __SHIFTIN(shared_mode, AE_MISC_CONTROL_ONE_CTX_RELOAD); 866 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val); 867 868 return 0; 869} 870 871static enum qat_ae_status 872qat_ae_get_status(struct qat_softc *sc, u_char ae) 873{ 874 int error; 875 uint32_t val = 0; 876 877 error = qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 878 if (error || val & CTX_ENABLES_ENABLE) 879 return QAT_AE_ENABLED; 880 881 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val); 882 if (val & ACTIVE_CTX_STATUS_ABO) 883 return QAT_AE_ACTIVE; 884 885 return QAT_AE_DISABLED; 886} 887 888 889static int 890qat_ae_is_active(struct qat_softc *sc, u_char ae) 891{ 892 uint32_t val; 893 894 if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED) 895 return 1; 896 897 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val); 898 if (val & ACTIVE_CTX_STATUS_ABO) 899 return 1; 900 else 901 return 0; 902} 903 904/* returns 1 if actually waited for specified number of cycles */ 905static int 906qat_ae_wait_num_cycles(struct qat_softc *sc, u_char ae, int cycles, int check) 907{ 908 uint32_t cnt, actx; 909 int pcnt, ccnt, elapsed, times; 910 911 qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); 912 pcnt = cnt & 0xffff; 913 914 times = TIMEOUT_AE_CHECK; 915 do { 916 qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); 917 ccnt = cnt & 0xffff; 918 919 elapsed = ccnt - pcnt; 920 if (elapsed == 0) { 921 times--; 922 } 923 if (times <= 0) { 924 device_printf(sc->sc_dev, 925 "qat_ae_wait_num_cycles timeout\n"); 926 return -1; 927 } 928 929 if (elapsed < 0) 930 elapsed += 0x10000; 931 932 if (elapsed >= CYCLES_FROM_READY2EXE && check) { 933 if (qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, 934 &actx) == 0) { 935 if ((actx & ACTIVE_CTX_STATUS_ABO) == 0) 936 return 0; 937 } 938 } 939 } while (cycles > elapsed); 940 941 if (check && qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) { 942 if ((actx & ACTIVE_CTX_STATUS_ABO) == 0) 943 return 0; 944 } 945 946 return 1; 947} 948 949int 950qat_ae_init(struct qat_softc *sc) 951{ 952 int error; 953 uint32_t mask, val = 0; 954 u_char ae; 955 956 /* XXX adf_initSysMemInfo */ 957 958 /* XXX Disable clock gating for some chip if debug mode */ 959 960 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 961 struct qat_ae *qae = &sc->sc_ae[ae]; 962 if (!(mask & 1)) 963 continue; 964 965 qae->qae_ustore_size = USTORE_SIZE; 966 967 qae->qae_free_addr = 0; 968 qae->qae_free_size = USTORE_SIZE; 969 qae->qae_live_ctx_mask = AE_ALL_CTX; 970 qae->qae_ustore_dram_addr = 0; 971 qae->qae_reload_size = 0; 972 } 973 974 /* XXX Enable attention interrupt */ 975 976 error = qat_ae_clear_reset(sc); 977 if (error) 978 return error; 979 980 qat_ae_clear_xfer(sc); 981 982 if (!sc->sc_hw.qhw_fw_auth) { 983 error = qat_ae_clear_gprs(sc); 984 if (error) 985 return error; 986 } 987 988 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ 989 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 990 if (!(mask & 1)) 991 continue; 992 qat_ae_read_4(sc, ae, SIGNATURE_ENABLE, &val); 993 val |= 0x1; 994 qat_ae_write_4(sc, ae, SIGNATURE_ENABLE, val); 995 } 996 997 error = qat_ae_clear_reset(sc); 998 if (error) 999 return error; 1000 1001 /* XXX XXX XXX Clean MMP memory if mem scrub is supported */ 1002 /* halMem_ScrubMMPMemory */ 1003 1004 return 0; 1005} 1006 1007int 1008qat_ae_start(struct qat_softc *sc) 1009{ 1010 int error; 1011 u_char ae; 1012 1013 for (ae = 0; ae < sc->sc_ae_num; ae++) { 1014 if ((sc->sc_ae_mask & (1 << ae)) == 0) 1015 continue; 1016 1017 error = qat_aefw_start(sc, ae, 0xff); 1018 if (error) 1019 return error; 1020 } 1021 1022 return 0; 1023} 1024 1025void 1026qat_ae_cluster_intr(void *arg) 1027{ 1028 /* Nothing to implement until we support SRIOV. */ 1029 printf("qat_ae_cluster_intr\n"); 1030} 1031 1032static int 1033qat_ae_clear_reset(struct qat_softc *sc) 1034{ 1035 int error; 1036 uint32_t times, reset, clock, reg, mask; 1037 u_char ae; 1038 1039 reset = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET); 1040 reset &= ~(__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK)); 1041 reset &= ~(__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK)); 1042 times = TIMEOUT_AE_RESET; 1043 do { 1044 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_RESET, reset); 1045 if ((times--) == 0) { 1046 device_printf(sc->sc_dev, "couldn't reset AEs\n"); 1047 return EBUSY; 1048 } 1049 reg = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET); 1050 } while ((__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK) | 1051 __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK)) 1052 & reg); 1053 1054 /* Enable clock for AE and QAT */ 1055 clock = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_CLK_EN); 1056 clock |= __SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_CLK_EN_AE_MASK); 1057 clock |= __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK); 1058 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_CLK_EN, clock); 1059 1060 error = qat_ae_check(sc); 1061 if (error) 1062 return error; 1063 1064 /* 1065 * Set undefined power-up/reset states to reasonable default values... 1066 * just to make sure we're starting from a known point 1067 */ 1068 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1069 if (!(mask & 1)) 1070 continue; 1071 1072 /* init the ctx_enable */ 1073 qat_ae_write_4(sc, ae, CTX_ENABLES, 1074 CTX_ENABLES_INIT); 1075 1076 /* initialize the PCs */ 1077 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1078 CTX_STS_INDIRECT, 1079 UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); 1080 1081 /* init the ctx_arb */ 1082 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, 1083 CTX_ARB_CNTL_INIT); 1084 1085 /* enable cc */ 1086 qat_ae_write_4(sc, ae, CC_ENABLE, 1087 CC_ENABLE_INIT); 1088 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1089 CTX_WAKEUP_EVENTS_INDIRECT, 1090 CTX_WAKEUP_EVENTS_INDIRECT_INIT); 1091 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1092 CTX_SIG_EVENTS_INDIRECT, 1093 CTX_SIG_EVENTS_INDIRECT_INIT); 1094 } 1095 1096 if ((sc->sc_ae_mask != 0) && 1097 sc->sc_flags & QAT_FLAG_ESRAM_ENABLE_AUTO_INIT) { 1098 /* XXX XXX XXX init eSram only when this is boot time */ 1099 } 1100 1101 if ((sc->sc_ae_mask != 0) && 1102 sc->sc_flags & QAT_FLAG_SHRAM_WAIT_READY) { 1103 /* XXX XXX XXX wait shram to complete initialization */ 1104 } 1105 1106 qat_ae_reset_timestamp(sc); 1107 1108 return 0; 1109} 1110 1111static int 1112qat_ae_check(struct qat_softc *sc) 1113{ 1114 int error, times, ae; 1115 uint32_t cnt, pcnt, mask; 1116 1117 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1118 if (!(mask & 1)) 1119 continue; 1120 1121 times = TIMEOUT_AE_CHECK; 1122 error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); 1123 if (error) { 1124 device_printf(sc->sc_dev, 1125 "couldn't access AE %d CSR\n", ae); 1126 return error; 1127 } 1128 pcnt = cnt & 0xffff; 1129 1130 while (1) { 1131 error = qat_ae_read_4(sc, ae, 1132 PROFILE_COUNT, &cnt); 1133 if (error) { 1134 device_printf(sc->sc_dev, 1135 "couldn't access AE %d CSR\n", ae); 1136 return error; 1137 } 1138 cnt &= 0xffff; 1139 if (cnt == pcnt) 1140 times--; 1141 else 1142 break; 1143 if (times <= 0) { 1144 device_printf(sc->sc_dev, 1145 "AE %d CSR is useless\n", ae); 1146 return EFAULT; 1147 } 1148 } 1149 } 1150 1151 return 0; 1152} 1153 1154static int 1155qat_ae_reset_timestamp(struct qat_softc *sc) 1156{ 1157 uint32_t misc, mask; 1158 u_char ae; 1159 1160 /* stop the timestamp timers */ 1161 misc = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_MISC); 1162 if (misc & CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN) { 1163 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC, 1164 misc & (~CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN)); 1165 } 1166 1167 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1168 if (!(mask & 1)) 1169 continue; 1170 qat_ae_write_4(sc, ae, TIMESTAMP_LOW, 0); 1171 qat_ae_write_4(sc, ae, TIMESTAMP_HIGH, 0); 1172 } 1173 1174 /* start timestamp timers */ 1175 qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC, 1176 misc | CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN); 1177 1178 return 0; 1179} 1180 1181static void 1182qat_ae_clear_xfer(struct qat_softc *sc) 1183{ 1184 u_int mask, reg; 1185 u_char ae; 1186 1187 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1188 if (!(mask & 1)) 1189 continue; 1190 1191 for (reg = 0; reg < MAX_GPR_REG; reg++) { 1192 qat_aereg_abs_data_write(sc, ae, AEREG_SR_RD_ABS, 1193 reg, 0); 1194 qat_aereg_abs_data_write(sc, ae, AEREG_DR_RD_ABS, 1195 reg, 0); 1196 } 1197 } 1198} 1199 1200static int 1201qat_ae_clear_gprs(struct qat_softc *sc) 1202{ 1203 uint32_t val; 1204 uint32_t saved_ctx = 0; 1205 int times = TIMEOUT_AE_CHECK, rv; 1206 u_char ae; 1207 u_int mask; 1208 1209 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1210 if (!(mask & 1)) 1211 continue; 1212 1213 /* turn off share control store bit */ 1214 val = qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); 1215 val &= ~AE_MISC_CONTROL_SHARE_CS; 1216 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val); 1217 1218 /* turn off ucode parity */ 1219 /* make sure nn_mode is set to self */ 1220 qat_ae_read_4(sc, ae, CTX_ENABLES, &val); 1221 val &= CTX_ENABLES_IGNORE_W1C_MASK; 1222 val |= CTX_ENABLES_NN_MODE; 1223 val &= ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE; 1224 qat_ae_write_4(sc, ae, CTX_ENABLES, val); 1225 1226 /* copy instructions to ustore */ 1227 qat_ae_ucode_write(sc, ae, 0, nitems(ae_clear_gprs_inst), 1228 ae_clear_gprs_inst); 1229 1230 /* set PC */ 1231 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT, 1232 UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); 1233 1234 /* save current context */ 1235 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &saved_ctx); 1236 /* change the active context */ 1237 /* start the context from ctx 0 */ 1238 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 0); 1239 1240 /* wakeup-event voluntary */ 1241 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1242 CTX_WAKEUP_EVENTS_INDIRECT, 1243 CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY); 1244 /* clean signals */ 1245 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1246 CTX_SIG_EVENTS_INDIRECT, 0); 1247 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0); 1248 1249 qat_ae_enable_ctx(sc, ae, AE_ALL_CTX); 1250 } 1251 1252 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 1253 if (!(mask & 1)) 1254 continue; 1255 /* wait for AE to finish */ 1256 do { 1257 rv = qat_ae_wait_num_cycles(sc, ae, AE_EXEC_CYCLE, 1); 1258 } while (rv && times--); 1259 if (times <= 0) { 1260 device_printf(sc->sc_dev, 1261 "qat_ae_clear_gprs timeout"); 1262 return ETIMEDOUT; 1263 } 1264 qat_ae_disable_ctx(sc, ae, AE_ALL_CTX); 1265 /* change the active context */ 1266 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 1267 saved_ctx & ACTIVE_CTX_STATUS_ACNO); 1268 /* init the ctx_enable */ 1269 qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT); 1270 /* initialize the PCs */ 1271 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1272 CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); 1273 /* init the ctx_arb */ 1274 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT); 1275 /* enable cc */ 1276 qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT); 1277 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, 1278 CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT); 1279 qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT, 1280 CTX_SIG_EVENTS_INDIRECT_INIT); 1281 } 1282 1283 return 0; 1284} 1285 1286static void 1287qat_ae_get_shared_ustore_ae(u_char ae, u_char *nae) 1288{ 1289 if (ae & 0x1) 1290 *nae = ae - 1; 1291 else 1292 *nae = ae + 1; 1293} 1294 1295static u_int 1296qat_ae_ucode_parity64(uint64_t ucode) 1297{ 1298 1299 ucode ^= ucode >> 1; 1300 ucode ^= ucode >> 2; 1301 ucode ^= ucode >> 4; 1302 ucode ^= ucode >> 8; 1303 ucode ^= ucode >> 16; 1304 ucode ^= ucode >> 32; 1305 1306 return ((u_int)(ucode & 1)); 1307} 1308 1309static uint64_t 1310qat_ae_ucode_set_ecc(uint64_t ucode) 1311{ 1312 static const uint64_t 1313 bit0mask=0xff800007fffULL, bit1mask=0x1f801ff801fULL, 1314 bit2mask=0xe387e0781e1ULL, bit3mask=0x7cb8e388e22ULL, 1315 bit4mask=0xaf5b2c93244ULL, bit5mask=0xf56d5525488ULL, 1316 bit6mask=0xdaf69a46910ULL; 1317 1318 /* clear the ecc bits */ 1319 ucode &= ~(0x7fULL << USTORE_ECC_BIT_0); 1320 1321 ucode |= (uint64_t)qat_ae_ucode_parity64(bit0mask & ucode) << 1322 USTORE_ECC_BIT_0; 1323 ucode |= (uint64_t)qat_ae_ucode_parity64(bit1mask & ucode) << 1324 USTORE_ECC_BIT_1; 1325 ucode |= (uint64_t)qat_ae_ucode_parity64(bit2mask & ucode) << 1326 USTORE_ECC_BIT_2; 1327 ucode |= (uint64_t)qat_ae_ucode_parity64(bit3mask & ucode) << 1328 USTORE_ECC_BIT_3; 1329 ucode |= (uint64_t)qat_ae_ucode_parity64(bit4mask & ucode) << 1330 USTORE_ECC_BIT_4; 1331 ucode |= (uint64_t)qat_ae_ucode_parity64(bit5mask & ucode) << 1332 USTORE_ECC_BIT_5; 1333 ucode |= (uint64_t)qat_ae_ucode_parity64(bit6mask & ucode) << 1334 USTORE_ECC_BIT_6; 1335 1336 return (ucode); 1337} 1338 1339static int 1340qat_ae_ucode_write(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst, 1341 const uint64_t *ucode) 1342{ 1343 uint64_t tmp; 1344 uint32_t ustore_addr, ulo, uhi; 1345 int i; 1346 1347 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); 1348 uaddr |= USTORE_ADDRESS_ECS; 1349 1350 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 1351 for (i = 0; i < ninst; i++) { 1352 tmp = qat_ae_ucode_set_ecc(ucode[i]); 1353 ulo = (uint32_t)(tmp & 0xffffffff); 1354 uhi = (uint32_t)(tmp >> 32); 1355 1356 qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo); 1357 /* this will auto increment the address */ 1358 qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi); 1359 } 1360 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); 1361 1362 return 0; 1363} 1364 1365static int 1366qat_ae_ucode_read(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst, 1367 uint64_t *ucode) 1368{ 1369 uint32_t misc, ustore_addr, ulo, uhi; 1370 u_int ii; 1371 u_char nae; 1372 1373 if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED) 1374 return EBUSY; 1375 1376 /* determine whether it neighbour AE runs in shared control store 1377 * status */ 1378 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 1379 if (misc & AE_MISC_CONTROL_SHARE_CS) { 1380 qat_ae_get_shared_ustore_ae(ae, &nae); 1381 if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae)) 1382 return EBUSY; 1383 } 1384 1385 /* if reloadable, then get it all from dram-ustore */ 1386 if (__SHIFTOUT(misc, AE_MISC_CONTROL_CS_RELOAD)) 1387 panic("notyet"); /* XXX getReloadUwords */ 1388 1389 /* disable SHARE_CS bit to workaround silicon bug */ 1390 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc & 0xfffffffb); 1391 1392 MPASS(uaddr + ninst <= USTORE_SIZE); 1393 1394 /* save ustore-addr csr */ 1395 qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); 1396 1397 uaddr |= USTORE_ADDRESS_ECS; /* enable ecs bit */ 1398 for (ii = 0; ii < ninst; ii++) { 1399 qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); 1400 1401 uaddr++; 1402 qat_ae_read_4(sc, ae, USTORE_DATA_LOWER, &ulo); 1403 qat_ae_read_4(sc, ae, USTORE_DATA_UPPER, &uhi); 1404 ucode[ii] = uhi; 1405 ucode[ii] = (ucode[ii] << 32) | ulo; 1406 } 1407 1408 /* restore SHARE_CS bit to workaround silicon bug */ 1409 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 1410 qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); 1411 1412 return 0; 1413} 1414 1415static u_int 1416qat_ae_concat_ucode(uint64_t *ucode, u_int ninst, u_int size, u_int addr, 1417 u_int *value) 1418{ 1419 const uint64_t *inst_arr; 1420 u_int ninst0, curvalue; 1421 int ii, vali, fixup, usize = 0; 1422 1423 if (size == 0) 1424 return 0; 1425 1426 ninst0 = ninst; 1427 vali = 0; 1428 curvalue = value[vali++]; 1429 1430 switch (size) { 1431 case 0x1: 1432 inst_arr = ae_inst_1b; 1433 usize = nitems(ae_inst_1b); 1434 break; 1435 case 0x2: 1436 inst_arr = ae_inst_2b; 1437 usize = nitems(ae_inst_2b); 1438 break; 1439 case 0x3: 1440 inst_arr = ae_inst_3b; 1441 usize = nitems(ae_inst_3b); 1442 break; 1443 default: 1444 inst_arr = ae_inst_4b; 1445 usize = nitems(ae_inst_4b); 1446 break; 1447 } 1448 1449 fixup = ninst; 1450 for (ii = 0; ii < usize; ii++) 1451 ucode[ninst++] = inst_arr[ii]; 1452 1453 INSERT_IMMED_GPRA_CONST(ucode[fixup], (addr)); 1454 fixup++; 1455 INSERT_IMMED_GPRA_CONST(ucode[fixup], 0); 1456 fixup++; 1457 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0)); 1458 fixup++; 1459 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16)); 1460 /* XXX fixup++ ? */ 1461 1462 if (size <= 0x4) 1463 return (ninst - ninst0); 1464 1465 size -= sizeof(u_int); 1466 while (size >= sizeof(u_int)) { 1467 curvalue = value[vali++]; 1468 fixup = ninst; 1469 ucode[ninst++] = ae_inst_4b[0x2]; 1470 ucode[ninst++] = ae_inst_4b[0x3]; 1471 ucode[ninst++] = ae_inst_4b[0x8]; 1472 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16)); 1473 fixup++; 1474 INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0)); 1475 /* XXX fixup++ ? */ 1476 1477 addr += sizeof(u_int); 1478 size -= sizeof(u_int); 1479 } 1480 /* call this function recusive when the left size less than 4 */ 1481 ninst += 1482 qat_ae_concat_ucode(ucode, ninst, size, addr, value + vali); 1483 1484 return (ninst - ninst0); 1485} 1486 1487static int 1488qat_ae_exec_ucode(struct qat_softc *sc, u_char ae, u_char ctx, 1489 uint64_t *ucode, u_int ninst, int cond_code_off, u_int max_cycles, 1490 u_int *endpc) 1491{ 1492 int error = 0, share_cs = 0; 1493 uint64_t savucode[MAX_EXEC_INST]; 1494 uint32_t indr_lm_addr_0, indr_lm_addr_1; 1495 uint32_t indr_lm_addr_byte_0, indr_lm_addr_byte_1; 1496 uint32_t indr_future_cnt_sig; 1497 uint32_t indr_sig, active_sig; 1498 uint32_t wakeup_ev, savpc, savcc, savctx, ctxarbctl; 1499 uint32_t misc, nmisc, ctxen; 1500 u_char nae; 1501 1502 MPASS(ninst <= USTORE_SIZE); 1503 1504 if (qat_ae_is_active(sc, ae)) 1505 return EBUSY; 1506 1507 /* save current LM addr */ 1508 qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_0_INDIRECT, &indr_lm_addr_0); 1509 qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_1_INDIRECT, &indr_lm_addr_1); 1510 qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, 1511 &indr_lm_addr_byte_0); 1512 qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, 1513 &indr_lm_addr_byte_1); 1514 1515 /* backup shared control store bit, and force AE to 1516 none-shared mode before executing ucode snippet */ 1517 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 1518 if (misc & AE_MISC_CONTROL_SHARE_CS) { 1519 share_cs = 1; 1520 qat_ae_get_shared_ustore_ae(ae, &nae); 1521 if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae)) 1522 return EBUSY; 1523 } 1524 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; 1525 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); 1526 1527 /* save current states: */ 1528 if (ninst <= MAX_EXEC_INST) { 1529 error = qat_ae_ucode_read(sc, ae, 0, ninst, savucode); 1530 if (error) { 1531 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); 1532 return error; 1533 } 1534 } 1535 1536 /* save wakeup-events */ 1537 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_WAKEUP_EVENTS_INDIRECT, 1538 &wakeup_ev); 1539 /* save PC */ 1540 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &savpc); 1541 savpc &= UPC_MASK; 1542 1543 /* save ctx enables */ 1544 qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); 1545 ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; 1546 /* save conditional-code */ 1547 qat_ae_read_4(sc, ae, CC_ENABLE, &savcc); 1548 /* save current context */ 1549 qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx); 1550 qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl); 1551 1552 /* save indirect csrs */ 1553 qat_ae_ctx_indr_read(sc, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, 1554 &indr_future_cnt_sig); 1555 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &indr_sig); 1556 qat_ae_read_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, &active_sig); 1557 1558 /* turn off ucode parity */ 1559 qat_ae_write_4(sc, ae, CTX_ENABLES, 1560 ctxen & ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE); 1561 1562 /* copy instructions to ustore */ 1563 qat_ae_ucode_write(sc, ae, 0, ninst, ucode); 1564 /* set PC */ 1565 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, 0); 1566 /* change the active context */ 1567 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 1568 ctx & ACTIVE_CTX_STATUS_ACNO); 1569 1570 if (cond_code_off) { 1571 /* disable conditional-code*/ 1572 qat_ae_write_4(sc, ae, CC_ENABLE, savcc & 0xffffdfff); 1573 } 1574 1575 /* wakeup-event voluntary */ 1576 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, 1577 CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY); 1578 1579 /* clean signals */ 1580 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 0); 1581 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0); 1582 1583 /* enable context */ 1584 qat_ae_enable_ctx(sc, ae, 1 << ctx); 1585 1586 /* wait for it to finish */ 1587 if (qat_ae_wait_num_cycles(sc, ae, max_cycles, 1) != 0) 1588 error = ETIMEDOUT; 1589 1590 /* see if we need to get the current PC */ 1591 if (endpc != NULL) { 1592 uint32_t ctx_status; 1593 1594 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, 1595 &ctx_status); 1596 *endpc = ctx_status & UPC_MASK; 1597 } 1598#if 0 1599 { 1600 uint32_t ctx_status; 1601 1602 qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, 1603 &ctx_status); 1604 printf("%s: endpc 0x%08x\n", __func__, 1605 ctx_status & UPC_MASK); 1606 } 1607#endif 1608 1609 /* retore to previous states: */ 1610 /* disable context */ 1611 qat_ae_disable_ctx(sc, ae, 1 << ctx); 1612 if (ninst <= MAX_EXEC_INST) { 1613 /* instructions */ 1614 qat_ae_ucode_write(sc, ae, 0, ninst, savucode); 1615 } 1616 /* wakeup-events */ 1617 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT, 1618 wakeup_ev); 1619 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, savpc); 1620 1621 /* only restore shared control store bit, 1622 other bit might be changed by AE code snippet */ 1623 qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); 1624 if (share_cs) 1625 nmisc = misc | AE_MISC_CONTROL_SHARE_CS; 1626 else 1627 nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; 1628 qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); 1629 /* conditional-code */ 1630 qat_ae_write_4(sc, ae, CC_ENABLE, savcc); 1631 /* change the active context */ 1632 qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 1633 savctx & ACTIVE_CTX_STATUS_ACNO); 1634 /* restore the nxt ctx to run */ 1635 qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); 1636 /* restore current LM addr */ 1637 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_0_INDIRECT, 1638 indr_lm_addr_0); 1639 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_1_INDIRECT, 1640 indr_lm_addr_1); 1641 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, 1642 indr_lm_addr_byte_0); 1643 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, 1644 indr_lm_addr_byte_1); 1645 1646 /* restore indirect csrs */ 1647 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, FUTURE_COUNT_SIGNAL_INDIRECT, 1648 indr_future_cnt_sig); 1649 qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 1650 indr_sig); 1651 qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, active_sig); 1652 1653 /* ctx-enables */ 1654 qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); 1655 1656 return error; 1657} 1658 1659static int 1660qat_ae_exec_ucode_init_lm(struct qat_softc *sc, u_char ae, u_char ctx, 1661 int *first_exec, uint64_t *ucode, u_int ninst, 1662 u_int *gpr_a0, u_int *gpr_a1, u_int *gpr_a2, u_int *gpr_b0, u_int *gpr_b1) 1663{ 1664 1665 if (*first_exec) { 1666 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0); 1667 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1); 1668 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2); 1669 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0); 1670 qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1); 1671 *first_exec = 0; 1672 } 1673 1674 return qat_ae_exec_ucode(sc, ae, ctx, ucode, ninst, 1, ninst * 5, NULL); 1675} 1676 1677static int 1678qat_ae_restore_init_lm_gprs(struct qat_softc *sc, u_char ae, u_char ctx, 1679 u_int gpr_a0, u_int gpr_a1, u_int gpr_a2, u_int gpr_b0, u_int gpr_b1) 1680{ 1681 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0); 1682 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1); 1683 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2); 1684 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0); 1685 qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1); 1686 1687 return 0; 1688} 1689 1690static int 1691qat_ae_get_inst_num(int lmsize) 1692{ 1693 int ninst, left; 1694 1695 if (lmsize == 0) 1696 return 0; 1697 1698 left = lmsize % sizeof(u_int); 1699 1700 if (left) { 1701 ninst = nitems(ae_inst_1b) + 1702 qat_ae_get_inst_num(lmsize - left); 1703 } else { 1704 /* 3 instruction is needed for further code */ 1705 ninst = (lmsize - sizeof(u_int)) * 3 / 4 + nitems(ae_inst_4b); 1706 } 1707 1708 return (ninst); 1709} 1710 1711static int 1712qat_ae_batch_put_lm(struct qat_softc *sc, u_char ae, 1713 struct qat_ae_batch_init_list *qabi_list, size_t nqabi) 1714{ 1715 struct qat_ae_batch_init *qabi; 1716 size_t alloc_ninst, ninst; 1717 uint64_t *ucode; 1718 u_int gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1; 1719 int insnsz, error = 0, execed = 0, first_exec = 1; 1720 1721 if (STAILQ_FIRST(qabi_list) == NULL) 1722 return 0; 1723 1724 alloc_ninst = min(USTORE_SIZE, nqabi); 1725 ucode = qat_alloc_mem(sizeof(uint64_t) * alloc_ninst); 1726 1727 ninst = 0; 1728 STAILQ_FOREACH(qabi, qabi_list, qabi_next) { 1729 insnsz = qat_ae_get_inst_num(qabi->qabi_size); 1730 if (insnsz + ninst > alloc_ninst) { 1731 /* add ctx_arb[kill] */ 1732 ucode[ninst++] = 0x0E000010000ull; 1733 execed = 1; 1734 1735 error = qat_ae_exec_ucode_init_lm(sc, ae, 0, 1736 &first_exec, ucode, ninst, 1737 &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1); 1738 if (error) { 1739 qat_ae_restore_init_lm_gprs(sc, ae, 0, 1740 gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1); 1741 qat_free_mem(ucode); 1742 return error; 1743 } 1744 /* run microExec to execute the microcode */ 1745 ninst = 0; 1746 } 1747 ninst += qat_ae_concat_ucode(ucode, ninst, 1748 qabi->qabi_size, qabi->qabi_addr, qabi->qabi_value); 1749 } 1750 1751 if (ninst > 0) { 1752 ucode[ninst++] = 0x0E000010000ull; 1753 execed = 1; 1754 1755 error = qat_ae_exec_ucode_init_lm(sc, ae, 0, 1756 &first_exec, ucode, ninst, 1757 &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1); 1758 } 1759 if (execed) { 1760 qat_ae_restore_init_lm_gprs(sc, ae, 0, 1761 gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1); 1762 } 1763 1764 qat_free_mem(ucode); 1765 1766 return error; 1767} 1768 1769static int 1770qat_ae_write_pc(struct qat_softc *sc, u_char ae, u_int ctx_mask, u_int upc) 1771{ 1772 1773 if (qat_ae_is_active(sc, ae)) 1774 return EBUSY; 1775 1776 qat_ae_ctx_indr_write(sc, ae, ctx_mask, CTX_STS_INDIRECT, 1777 UPC_MASK & upc); 1778 return 0; 1779} 1780 1781static inline u_int 1782qat_aefw_csum_calc(u_int reg, int ch) 1783{ 1784 int i; 1785 u_int topbit = CRC_BITMASK(CRC_WIDTH - 1); 1786 u_int inbyte = (u_int)((reg >> 0x18) ^ ch); 1787 1788 reg ^= inbyte << (CRC_WIDTH - 0x8); 1789 for (i = 0; i < 0x8; i++) { 1790 if (reg & topbit) 1791 reg = (reg << 1) ^ CRC_POLY; 1792 else 1793 reg <<= 1; 1794 } 1795 1796 return (reg & CRC_WIDTHMASK(CRC_WIDTH)); 1797} 1798 1799static u_int 1800qat_aefw_csum(char *buf, int size) 1801{ 1802 u_int csum = 0; 1803 1804 while (size--) { 1805 csum = qat_aefw_csum_calc(csum, *buf++); 1806 } 1807 1808 return csum; 1809} 1810 1811static const char * 1812qat_aefw_uof_string(struct qat_softc *sc, size_t offset) 1813{ 1814 if (offset >= sc->sc_aefw_uof.qafu_str_tab_size) 1815 return NULL; 1816 if (sc->sc_aefw_uof.qafu_str_tab == NULL) 1817 return NULL; 1818 1819 return (const char *)((uintptr_t)sc->sc_aefw_uof.qafu_str_tab + offset); 1820} 1821 1822static struct uof_chunk_hdr * 1823qat_aefw_uof_find_chunk(struct qat_softc *sc, 1824 const char *id, struct uof_chunk_hdr *cur) 1825{ 1826 struct uof_obj_hdr *uoh = sc->sc_aefw_uof.qafu_obj_hdr; 1827 struct uof_chunk_hdr *uch; 1828 int i; 1829 1830 uch = (struct uof_chunk_hdr *)(uoh + 1); 1831 for (i = 0; i < uoh->uoh_num_chunks; i++, uch++) { 1832 if (uch->uch_offset + uch->uch_size > sc->sc_aefw_uof.qafu_size) 1833 return NULL; 1834 1835 if (cur < uch && !strncmp(uch->uch_id, id, UOF_OBJ_ID_LEN)) 1836 return uch; 1837 } 1838 1839 return NULL; 1840} 1841 1842static int 1843qat_aefw_load_mof(struct qat_softc *sc) 1844{ 1845 const struct firmware *fw; 1846 1847 fw = firmware_get(sc->sc_hw.qhw_mof_fwname); 1848 if (fw == NULL) { 1849 device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n", 1850 sc->sc_hw.qhw_mof_fwname); 1851 return ENXIO; 1852 } 1853 1854 sc->sc_fw_mof = qat_alloc_mem(fw->datasize); 1855 sc->sc_fw_mof_size = fw->datasize; 1856 memcpy(sc->sc_fw_mof, fw->data, fw->datasize); 1857 firmware_put(fw, FIRMWARE_UNLOAD); 1858 return 0; 1859} 1860 1861static void 1862qat_aefw_unload_mof(struct qat_softc *sc) 1863{ 1864 if (sc->sc_fw_mof != NULL) { 1865 qat_free_mem(sc->sc_fw_mof); 1866 sc->sc_fw_mof = NULL; 1867 } 1868} 1869 1870static int 1871qat_aefw_load_mmp(struct qat_softc *sc) 1872{ 1873 const struct firmware *fw; 1874 1875 fw = firmware_get(sc->sc_hw.qhw_mmp_fwname); 1876 if (fw == NULL) { 1877 device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n", 1878 sc->sc_hw.qhw_mmp_fwname); 1879 return ENXIO; 1880 } 1881 1882 sc->sc_fw_mmp = qat_alloc_mem(fw->datasize); 1883 sc->sc_fw_mmp_size = fw->datasize; 1884 memcpy(sc->sc_fw_mmp, fw->data, fw->datasize); 1885 firmware_put(fw, FIRMWARE_UNLOAD); 1886 return 0; 1887} 1888 1889static void 1890qat_aefw_unload_mmp(struct qat_softc *sc) 1891{ 1892 if (sc->sc_fw_mmp != NULL) { 1893 qat_free_mem(sc->sc_fw_mmp); 1894 sc->sc_fw_mmp = NULL; 1895 } 1896} 1897 1898static int 1899qat_aefw_mof_find_uof0(struct qat_softc *sc, 1900 struct mof_uof_hdr *muh, struct mof_uof_chunk_hdr *head, 1901 u_int nchunk, size_t size, const char *id, 1902 size_t *fwsize, void **fwptr) 1903{ 1904 int i; 1905 char *uof_name; 1906 1907 for (i = 0; i < nchunk; i++) { 1908 struct mof_uof_chunk_hdr *much = &head[i]; 1909 1910 if (strncmp(much->much_id, id, MOF_OBJ_ID_LEN)) 1911 return EINVAL; 1912 1913 if (much->much_offset + much->much_size > size) 1914 return EINVAL; 1915 1916 if (sc->sc_mof.qmf_sym_size <= much->much_name) 1917 return EINVAL; 1918 1919 uof_name = (char *)((uintptr_t)sc->sc_mof.qmf_sym + 1920 much->much_name); 1921 1922 if (!strcmp(uof_name, sc->sc_fw_uof_name)) { 1923 *fwptr = (void *)((uintptr_t)muh + 1924 (uintptr_t)much->much_offset); 1925 *fwsize = (size_t)much->much_size; 1926 return 0; 1927 } 1928 } 1929 1930 return ENOENT; 1931} 1932 1933static int 1934qat_aefw_mof_find_uof(struct qat_softc *sc) 1935{ 1936 struct mof_uof_hdr *uof_hdr, *suof_hdr; 1937 u_int nuof_chunks = 0, nsuof_chunks = 0; 1938 int error; 1939 1940 uof_hdr = sc->sc_mof.qmf_uof_objs; 1941 suof_hdr = sc->sc_mof.qmf_suof_objs; 1942 1943 if (uof_hdr != NULL) { 1944 if (uof_hdr->muh_max_chunks < uof_hdr->muh_num_chunks) { 1945 return EINVAL; 1946 } 1947 nuof_chunks = uof_hdr->muh_num_chunks; 1948 } 1949 if (suof_hdr != NULL) { 1950 if (suof_hdr->muh_max_chunks < suof_hdr->muh_num_chunks) 1951 return EINVAL; 1952 nsuof_chunks = suof_hdr->muh_num_chunks; 1953 } 1954 1955 if (nuof_chunks + nsuof_chunks == 0) 1956 return EINVAL; 1957 1958 if (uof_hdr != NULL) { 1959 error = qat_aefw_mof_find_uof0(sc, uof_hdr, 1960 (struct mof_uof_chunk_hdr *)(uof_hdr + 1), nuof_chunks, 1961 sc->sc_mof.qmf_uof_objs_size, UOF_IMAG, 1962 &sc->sc_fw_uof_size, &sc->sc_fw_uof); 1963 if (error && error != ENOENT) 1964 return error; 1965 } 1966 1967 if (suof_hdr != NULL) { 1968 error = qat_aefw_mof_find_uof0(sc, suof_hdr, 1969 (struct mof_uof_chunk_hdr *)(suof_hdr + 1), nsuof_chunks, 1970 sc->sc_mof.qmf_suof_objs_size, SUOF_IMAG, 1971 &sc->sc_fw_suof_size, &sc->sc_fw_suof); 1972 if (error && error != ENOENT) 1973 return error; 1974 } 1975 1976 if (sc->sc_fw_uof == NULL && sc->sc_fw_suof == NULL) 1977 return ENOENT; 1978 1979 return 0; 1980} 1981 1982static int 1983qat_aefw_mof_parse(struct qat_softc *sc) 1984{ 1985 const struct mof_file_hdr *mfh; 1986 const struct mof_file_chunk_hdr *mfch; 1987 size_t size; 1988 u_int csum; 1989 int error, i; 1990 1991 size = sc->sc_fw_mof_size; 1992 1993 if (size < sizeof(struct mof_file_hdr)) 1994 return EINVAL; 1995 size -= sizeof(struct mof_file_hdr); 1996 1997 mfh = sc->sc_fw_mof; 1998 1999 if (mfh->mfh_fid != MOF_FID) 2000 return EINVAL; 2001 2002 csum = qat_aefw_csum((char *)((uintptr_t)sc->sc_fw_mof + 2003 offsetof(struct mof_file_hdr, mfh_min_ver)), 2004 sc->sc_fw_mof_size - 2005 offsetof(struct mof_file_hdr, mfh_min_ver)); 2006 if (mfh->mfh_csum != csum) 2007 return EINVAL; 2008 2009 if (mfh->mfh_min_ver != MOF_MIN_VER || 2010 mfh->mfh_maj_ver != MOF_MAJ_VER) 2011 return EINVAL; 2012 2013 if (mfh->mfh_max_chunks < mfh->mfh_num_chunks) 2014 return EINVAL; 2015 2016 if (size < sizeof(struct mof_file_chunk_hdr) * mfh->mfh_num_chunks) 2017 return EINVAL; 2018 mfch = (const struct mof_file_chunk_hdr *)(mfh + 1); 2019 2020 for (i = 0; i < mfh->mfh_num_chunks; i++, mfch++) { 2021 if (mfch->mfch_offset + mfch->mfch_size > sc->sc_fw_mof_size) 2022 return EINVAL; 2023 2024 if (!strncmp(mfch->mfch_id, SYM_OBJS, MOF_OBJ_ID_LEN)) { 2025 if (sc->sc_mof.qmf_sym != NULL) 2026 return EINVAL; 2027 2028 sc->sc_mof.qmf_sym = 2029 (void *)((uintptr_t)sc->sc_fw_mof + 2030 (uintptr_t)mfch->mfch_offset + sizeof(u_int)); 2031 sc->sc_mof.qmf_sym_size = 2032 *(u_int *)((uintptr_t)sc->sc_fw_mof + 2033 (uintptr_t)mfch->mfch_offset); 2034 2035 if (sc->sc_mof.qmf_sym_size % sizeof(u_int) != 0) 2036 return EINVAL; 2037 if (mfch->mfch_size != sc->sc_mof.qmf_sym_size + 2038 sizeof(u_int) || mfch->mfch_size == 0) 2039 return EINVAL; 2040 if (*(char *)((uintptr_t)sc->sc_mof.qmf_sym + 2041 sc->sc_mof.qmf_sym_size - 1) != '\0') 2042 return EINVAL; 2043 2044 } else if (!strncmp(mfch->mfch_id, UOF_OBJS, MOF_OBJ_ID_LEN)) { 2045 if (sc->sc_mof.qmf_uof_objs != NULL) 2046 return EINVAL; 2047 2048 sc->sc_mof.qmf_uof_objs = 2049 (void *)((uintptr_t)sc->sc_fw_mof + 2050 (uintptr_t)mfch->mfch_offset); 2051 sc->sc_mof.qmf_uof_objs_size = mfch->mfch_size; 2052 2053 } else if (!strncmp(mfch->mfch_id, SUOF_OBJS, MOF_OBJ_ID_LEN)) { 2054 if (sc->sc_mof.qmf_suof_objs != NULL) 2055 return EINVAL; 2056 2057 sc->sc_mof.qmf_suof_objs = 2058 (void *)((uintptr_t)sc->sc_fw_mof + 2059 (uintptr_t)mfch->mfch_offset); 2060 sc->sc_mof.qmf_suof_objs_size = mfch->mfch_size; 2061 } 2062 } 2063 2064 if (sc->sc_mof.qmf_sym == NULL || 2065 (sc->sc_mof.qmf_uof_objs == NULL && 2066 sc->sc_mof.qmf_suof_objs == NULL)) 2067 return EINVAL; 2068 2069 error = qat_aefw_mof_find_uof(sc); 2070 if (error) 2071 return error; 2072 return 0; 2073} 2074 2075static int 2076qat_aefw_uof_parse_image(struct qat_softc *sc, 2077 struct qat_uof_image *qui, struct uof_chunk_hdr *uch) 2078{ 2079 struct uof_image *image; 2080 struct uof_code_page *page; 2081 uintptr_t base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr; 2082 size_t lim = uch->uch_offset + uch->uch_size, size; 2083 int i, p; 2084 2085 size = uch->uch_size; 2086 if (size < sizeof(struct uof_image)) 2087 return EINVAL; 2088 size -= sizeof(struct uof_image); 2089 2090 qui->qui_image = image = 2091 (struct uof_image *)(base + uch->uch_offset); 2092 2093#define ASSIGN_OBJ_TAB(np, typep, type, base, off, lim) \ 2094do { \ 2095 u_int nent; \ 2096 nent = ((struct uof_obj_table *)((base) + (off)))->uot_nentries;\ 2097 if ((lim) < off + sizeof(struct uof_obj_table) + \ 2098 sizeof(type) * nent) \ 2099 return EINVAL; \ 2100 *(np) = nent; \ 2101 if (nent > 0) \ 2102 *(typep) = (type)((struct uof_obj_table *) \ 2103 ((base) + (off)) + 1); \ 2104 else \ 2105 *(typep) = NULL; \ 2106} while (0) 2107 2108 ASSIGN_OBJ_TAB(&qui->qui_num_ae_reg, &qui->qui_ae_reg, 2109 struct uof_ae_reg *, base, image->ui_reg_tab, lim); 2110 ASSIGN_OBJ_TAB(&qui->qui_num_init_reg_sym, &qui->qui_init_reg_sym, 2111 struct uof_init_reg_sym *, base, image->ui_init_reg_sym_tab, lim); 2112 ASSIGN_OBJ_TAB(&qui->qui_num_sbreak, &qui->qui_sbreak, 2113 struct qui_sbreak *, base, image->ui_sbreak_tab, lim); 2114 2115 if (size < sizeof(struct uof_code_page) * image->ui_num_pages) 2116 return EINVAL; 2117 if (nitems(qui->qui_pages) < image->ui_num_pages) 2118 return EINVAL; 2119 2120 page = (struct uof_code_page *)(image + 1); 2121 2122 for (p = 0; p < image->ui_num_pages; p++, page++) { 2123 struct qat_uof_page *qup = &qui->qui_pages[p]; 2124 struct uof_code_area *uca; 2125 2126 qup->qup_page_num = page->ucp_page_num; 2127 qup->qup_def_page = page->ucp_def_page; 2128 qup->qup_page_region = page->ucp_page_region; 2129 qup->qup_beg_vaddr = page->ucp_beg_vaddr; 2130 qup->qup_beg_paddr = page->ucp_beg_paddr; 2131 2132 ASSIGN_OBJ_TAB(&qup->qup_num_uc_var, &qup->qup_uc_var, 2133 struct uof_uword_fixup *, base, 2134 page->ucp_uc_var_tab, lim); 2135 ASSIGN_OBJ_TAB(&qup->qup_num_imp_var, &qup->qup_imp_var, 2136 struct uof_import_var *, base, 2137 page->ucp_imp_var_tab, lim); 2138 ASSIGN_OBJ_TAB(&qup->qup_num_imp_expr, &qup->qup_imp_expr, 2139 struct uof_uword_fixup *, base, 2140 page->ucp_imp_expr_tab, lim); 2141 ASSIGN_OBJ_TAB(&qup->qup_num_neigh_reg, &qup->qup_neigh_reg, 2142 struct uof_uword_fixup *, base, 2143 page->ucp_neigh_reg_tab, lim); 2144 2145 if (lim < page->ucp_code_area + sizeof(struct uof_code_area)) 2146 return EINVAL; 2147 2148 uca = (struct uof_code_area *)(base + page->ucp_code_area); 2149 qup->qup_num_micro_words = uca->uca_num_micro_words; 2150 2151 ASSIGN_OBJ_TAB(&qup->qup_num_uw_blocks, &qup->qup_uw_blocks, 2152 struct qat_uof_uword_block *, base, 2153 uca->uca_uword_block_tab, lim); 2154 2155 for (i = 0; i < qup->qup_num_uw_blocks; i++) { 2156 u_int uwordoff = ((struct uof_uword_block *)( 2157 &qup->qup_uw_blocks[i]))->uub_uword_offset; 2158 2159 if (lim < uwordoff) 2160 return EINVAL; 2161 2162 qup->qup_uw_blocks[i].quub_micro_words = 2163 (base + uwordoff); 2164 } 2165 } 2166 2167#undef ASSIGN_OBJ_TAB 2168 2169 return 0; 2170} 2171 2172static int 2173qat_aefw_uof_parse_images(struct qat_softc *sc) 2174{ 2175 struct uof_chunk_hdr *uch = NULL; 2176 u_int assigned_ae; 2177 int i, error; 2178 2179 for (i = 0; i < MAX_NUM_AE * MAX_AE_CTX; i++) { 2180 uch = qat_aefw_uof_find_chunk(sc, UOF_IMAG, uch); 2181 if (uch == NULL) 2182 break; 2183 2184 if (i >= nitems(sc->sc_aefw_uof.qafu_imgs)) 2185 return ENOENT; 2186 2187 error = qat_aefw_uof_parse_image(sc, &sc->sc_aefw_uof.qafu_imgs[i], uch); 2188 if (error) 2189 return error; 2190 2191 sc->sc_aefw_uof.qafu_num_imgs++; 2192 } 2193 2194 assigned_ae = 0; 2195 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { 2196 assigned_ae |= sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned; 2197 } 2198 2199 return 0; 2200} 2201 2202static int 2203qat_aefw_uof_parse(struct qat_softc *sc) 2204{ 2205 struct uof_file_hdr *ufh; 2206 struct uof_file_chunk_hdr *ufch; 2207 struct uof_obj_hdr *uoh; 2208 struct uof_chunk_hdr *uch; 2209 void *uof = NULL; 2210 size_t size, uof_size, hdr_size; 2211 uintptr_t base; 2212 u_int csum; 2213 int i; 2214 2215 size = sc->sc_fw_uof_size; 2216 if (size < MIN_UOF_SIZE) 2217 return EINVAL; 2218 size -= sizeof(struct uof_file_hdr); 2219 2220 ufh = sc->sc_fw_uof; 2221 2222 if (ufh->ufh_id != UOF_FID) 2223 return EINVAL; 2224 if (ufh->ufh_min_ver != UOF_MIN_VER || ufh->ufh_maj_ver != UOF_MAJ_VER) 2225 return EINVAL; 2226 2227 if (ufh->ufh_max_chunks < ufh->ufh_num_chunks) 2228 return EINVAL; 2229 if (size < sizeof(struct uof_file_chunk_hdr) * ufh->ufh_num_chunks) 2230 return EINVAL; 2231 ufch = (struct uof_file_chunk_hdr *)(ufh + 1); 2232 2233 uof_size = 0; 2234 for (i = 0; i < ufh->ufh_num_chunks; i++, ufch++) { 2235 if (ufch->ufch_offset + ufch->ufch_size > sc->sc_fw_uof_size) 2236 return EINVAL; 2237 2238 if (!strncmp(ufch->ufch_id, UOF_OBJS, UOF_OBJ_ID_LEN)) { 2239 if (uof != NULL) 2240 return EINVAL; 2241 2242 uof = 2243 (void *)((uintptr_t)sc->sc_fw_uof + 2244 ufch->ufch_offset); 2245 uof_size = ufch->ufch_size; 2246 2247 csum = qat_aefw_csum(uof, uof_size); 2248 if (csum != ufch->ufch_csum) 2249 return EINVAL; 2250 } 2251 } 2252 2253 if (uof == NULL) 2254 return ENOENT; 2255 2256 size = uof_size; 2257 if (size < sizeof(struct uof_obj_hdr)) 2258 return EINVAL; 2259 size -= sizeof(struct uof_obj_hdr); 2260 2261 uoh = uof; 2262 2263 if (size < sizeof(struct uof_chunk_hdr) * uoh->uoh_num_chunks) 2264 return EINVAL; 2265 2266 /* Check if the UOF objects are compatible with the chip */ 2267 if ((uoh->uoh_cpu_type & sc->sc_hw.qhw_prod_type) == 0) 2268 return ENOTSUP; 2269 2270 if (uoh->uoh_min_cpu_ver > sc->sc_rev || 2271 uoh->uoh_max_cpu_ver < sc->sc_rev) 2272 return ENOTSUP; 2273 2274 sc->sc_aefw_uof.qafu_size = uof_size; 2275 sc->sc_aefw_uof.qafu_obj_hdr = uoh; 2276 2277 base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr; 2278 2279 /* map uof string-table */ 2280 uch = qat_aefw_uof_find_chunk(sc, UOF_STRT, NULL); 2281 if (uch != NULL) { 2282 hdr_size = offsetof(struct uof_str_tab, ust_strings); 2283 sc->sc_aefw_uof.qafu_str_tab = 2284 (void *)(base + uch->uch_offset + hdr_size); 2285 sc->sc_aefw_uof.qafu_str_tab_size = uch->uch_size - hdr_size; 2286 } 2287 2288 /* get ustore mem inits table -- should be only one */ 2289 uch = qat_aefw_uof_find_chunk(sc, UOF_IMEM, NULL); 2290 if (uch != NULL) { 2291 if (uch->uch_size < sizeof(struct uof_obj_table)) 2292 return EINVAL; 2293 sc->sc_aefw_uof.qafu_num_init_mem = ((struct uof_obj_table *)(base + 2294 uch->uch_offset))->uot_nentries; 2295 if (sc->sc_aefw_uof.qafu_num_init_mem) { 2296 sc->sc_aefw_uof.qafu_init_mem = 2297 (struct uof_init_mem *)(base + uch->uch_offset + 2298 sizeof(struct uof_obj_table)); 2299 sc->sc_aefw_uof.qafu_init_mem_size = 2300 uch->uch_size - sizeof(struct uof_obj_table); 2301 } 2302 } 2303 2304 uch = qat_aefw_uof_find_chunk(sc, UOF_MSEG, NULL); 2305 if (uch != NULL) { 2306 if (uch->uch_size < sizeof(struct uof_obj_table) + 2307 sizeof(struct uof_var_mem_seg)) 2308 return EINVAL; 2309 sc->sc_aefw_uof.qafu_var_mem_seg = 2310 (struct uof_var_mem_seg *)(base + uch->uch_offset + 2311 sizeof(struct uof_obj_table)); 2312 } 2313 2314 return qat_aefw_uof_parse_images(sc); 2315} 2316 2317static int 2318qat_aefw_suof_parse_image(struct qat_softc *sc, struct qat_suof_image *qsi, 2319 struct suof_chunk_hdr *sch) 2320{ 2321 struct qat_aefw_suof *qafs = &sc->sc_aefw_suof; 2322 struct simg_ae_mode *ae_mode; 2323 u_int maj_ver; 2324 2325 qsi->qsi_simg_buf = qafs->qafs_suof_buf + sch->sch_offset + 2326 sizeof(struct suof_obj_hdr); 2327 qsi->qsi_simg_len = 2328 ((struct suof_obj_hdr *) 2329 (qafs->qafs_suof_buf + sch->sch_offset))->soh_img_length; 2330 2331 qsi->qsi_css_header = qsi->qsi_simg_buf; 2332 qsi->qsi_css_key = qsi->qsi_css_header + sizeof(struct css_hdr); 2333 qsi->qsi_css_signature = qsi->qsi_css_key + 2334 CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN; 2335 qsi->qsi_css_simg = qsi->qsi_css_signature + CSS_SIGNATURE_LEN; 2336 2337 ae_mode = (struct simg_ae_mode *)qsi->qsi_css_simg; 2338 qsi->qsi_ae_mask = ae_mode->sam_ae_mask; 2339 qsi->qsi_simg_name = (u_long)&ae_mode->sam_simg_name; 2340 qsi->qsi_appmeta_data = (u_long)&ae_mode->sam_appmeta_data; 2341 qsi->qsi_fw_type = ae_mode->sam_fw_type; 2342 2343 if (ae_mode->sam_dev_type != sc->sc_hw.qhw_prod_type) 2344 return EINVAL; 2345 2346 maj_ver = (QAT_PID_MAJOR_REV | (sc->sc_rev & QAT_PID_MINOR_REV)) & 0xff; 2347 if ((maj_ver > ae_mode->sam_devmax_ver) || 2348 (maj_ver < ae_mode->sam_devmin_ver)) { 2349 return EINVAL; 2350 } 2351 2352 return 0; 2353} 2354 2355static int 2356qat_aefw_suof_parse(struct qat_softc *sc) 2357{ 2358 struct suof_file_hdr *sfh; 2359 struct suof_chunk_hdr *sch; 2360 struct qat_aefw_suof *qafs = &sc->sc_aefw_suof; 2361 struct qat_suof_image *qsi; 2362 size_t size; 2363 u_int csum; 2364 int ae0_img = MAX_AE; 2365 int i, error; 2366 2367 size = sc->sc_fw_suof_size; 2368 if (size < sizeof(struct suof_file_hdr)) 2369 return EINVAL; 2370 2371 sfh = sc->sc_fw_suof; 2372 2373 if (sfh->sfh_file_id != SUOF_FID) 2374 return EINVAL; 2375 if (sfh->sfh_fw_type != 0) 2376 return EINVAL; 2377 if (sfh->sfh_num_chunks <= 1) 2378 return EINVAL; 2379 if (sfh->sfh_min_ver != SUOF_MIN_VER || 2380 sfh->sfh_maj_ver != SUOF_MAJ_VER) 2381 return EINVAL; 2382 2383 csum = qat_aefw_csum((char *)&sfh->sfh_min_ver, 2384 size - offsetof(struct suof_file_hdr, sfh_min_ver)); 2385 if (csum != sfh->sfh_check_sum) 2386 return EINVAL; 2387 2388 size -= sizeof(struct suof_file_hdr); 2389 2390 qafs->qafs_file_id = SUOF_FID; 2391 qafs->qafs_suof_buf = sc->sc_fw_suof; 2392 qafs->qafs_suof_size = sc->sc_fw_suof_size; 2393 qafs->qafs_check_sum = sfh->sfh_check_sum; 2394 qafs->qafs_min_ver = sfh->sfh_min_ver; 2395 qafs->qafs_maj_ver = sfh->sfh_maj_ver; 2396 qafs->qafs_fw_type = sfh->sfh_fw_type; 2397 2398 if (size < sizeof(struct suof_chunk_hdr)) 2399 return EINVAL; 2400 sch = (struct suof_chunk_hdr *)(sfh + 1); 2401 size -= sizeof(struct suof_chunk_hdr); 2402 2403 if (size < sizeof(struct suof_str_tab)) 2404 return EINVAL; 2405 size -= offsetof(struct suof_str_tab, sst_strings); 2406 2407 qafs->qafs_sym_size = ((struct suof_str_tab *) 2408 (qafs->qafs_suof_buf + sch->sch_offset))->sst_tab_length; 2409 if (size < qafs->qafs_sym_size) 2410 return EINVAL; 2411 qafs->qafs_sym_str = qafs->qafs_suof_buf + sch->sch_offset + 2412 offsetof(struct suof_str_tab, sst_strings); 2413 2414 qafs->qafs_num_simgs = sfh->sfh_num_chunks - 1; 2415 if (qafs->qafs_num_simgs == 0) 2416 return EINVAL; 2417 2418 qsi = qat_alloc_mem( 2419 sizeof(struct qat_suof_image) * qafs->qafs_num_simgs); 2420 qafs->qafs_simg = qsi; 2421 2422 for (i = 0; i < qafs->qafs_num_simgs; i++) { 2423 error = qat_aefw_suof_parse_image(sc, &qsi[i], &sch[i + 1]); 2424 if (error) 2425 return error; 2426 if ((qsi[i].qsi_ae_mask & 0x1) != 0) 2427 ae0_img = i; 2428 } 2429 2430 if (ae0_img != qafs->qafs_num_simgs - 1) { 2431 struct qat_suof_image last_qsi; 2432 2433 memcpy(&last_qsi, &qsi[qafs->qafs_num_simgs - 1], 2434 sizeof(struct qat_suof_image)); 2435 memcpy(&qsi[qafs->qafs_num_simgs - 1], &qsi[ae0_img], 2436 sizeof(struct qat_suof_image)); 2437 memcpy(&qsi[ae0_img], &last_qsi, 2438 sizeof(struct qat_suof_image)); 2439 } 2440 2441 return 0; 2442} 2443 2444static int 2445qat_aefw_alloc_auth_dmamem(struct qat_softc *sc, char *image, size_t size, 2446 struct qat_dmamem *dma) 2447{ 2448 struct css_hdr *css = (struct css_hdr *)image; 2449 struct auth_chunk *auth_chunk; 2450 struct fw_auth_desc *auth_desc; 2451 size_t mapsize, simg_offset = sizeof(struct auth_chunk); 2452 bus_size_t bus_addr; 2453 uintptr_t virt_addr; 2454 int error; 2455 2456 if (size > AE_IMG_OFFSET + CSS_MAX_IMAGE_LEN) 2457 return EINVAL; 2458 2459 mapsize = (css->css_fw_type == CSS_AE_FIRMWARE) ? 2460 CSS_AE_SIMG_LEN + simg_offset : 2461 size + CSS_FWSK_PAD_LEN + simg_offset; 2462 error = qat_alloc_dmamem(sc, dma, 1, mapsize, PAGE_SIZE); 2463 if (error) 2464 return error; 2465 2466 memset(dma->qdm_dma_vaddr, 0, mapsize); 2467 2468 auth_chunk = dma->qdm_dma_vaddr; 2469 auth_chunk->ac_chunk_size = mapsize; 2470 auth_chunk->ac_chunk_bus_addr = dma->qdm_dma_seg.ds_addr; 2471 2472 virt_addr = (uintptr_t)dma->qdm_dma_vaddr; 2473 virt_addr += simg_offset; 2474 bus_addr = auth_chunk->ac_chunk_bus_addr; 2475 bus_addr += simg_offset; 2476 2477 auth_desc = &auth_chunk->ac_fw_auth_desc; 2478 auth_desc->fad_css_hdr_high = (uint64_t)bus_addr >> 32; 2479 auth_desc->fad_css_hdr_low = bus_addr; 2480 2481 memcpy((void *)virt_addr, image, sizeof(struct css_hdr)); 2482 /* pub key */ 2483 virt_addr += sizeof(struct css_hdr); 2484 bus_addr += sizeof(struct css_hdr); 2485 image += sizeof(struct css_hdr); 2486 2487 auth_desc->fad_fwsk_pub_high = (uint64_t)bus_addr >> 32; 2488 auth_desc->fad_fwsk_pub_low = bus_addr; 2489 2490 memcpy((void *)virt_addr, image, CSS_FWSK_MODULUS_LEN); 2491 memset((void *)(virt_addr + CSS_FWSK_MODULUS_LEN), 0, CSS_FWSK_PAD_LEN); 2492 memcpy((void *)(virt_addr + CSS_FWSK_MODULUS_LEN + CSS_FWSK_PAD_LEN), 2493 image + CSS_FWSK_MODULUS_LEN, sizeof(uint32_t)); 2494 2495 virt_addr += CSS_FWSK_PUB_LEN; 2496 bus_addr += CSS_FWSK_PUB_LEN; 2497 image += CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN; 2498 2499 auth_desc->fad_signature_high = (uint64_t)bus_addr >> 32; 2500 auth_desc->fad_signature_low = bus_addr; 2501 2502 memcpy((void *)virt_addr, image, CSS_SIGNATURE_LEN); 2503 2504 virt_addr += CSS_SIGNATURE_LEN; 2505 bus_addr += CSS_SIGNATURE_LEN; 2506 image += CSS_SIGNATURE_LEN; 2507 2508 auth_desc->fad_img_high = (uint64_t)bus_addr >> 32; 2509 auth_desc->fad_img_low = bus_addr; 2510 auth_desc->fad_img_len = size - AE_IMG_OFFSET; 2511 2512 memcpy((void *)virt_addr, image, auth_desc->fad_img_len); 2513 2514 if (css->css_fw_type == CSS_AE_FIRMWARE) { 2515 auth_desc->fad_img_ae_mode_data_high = auth_desc->fad_img_high; 2516 auth_desc->fad_img_ae_mode_data_low = auth_desc->fad_img_low; 2517 2518 bus_addr += sizeof(struct simg_ae_mode); 2519 2520 auth_desc->fad_img_ae_init_data_high = (uint64_t)bus_addr >> 32; 2521 auth_desc->fad_img_ae_init_data_low = bus_addr; 2522 2523 bus_addr += SIMG_AE_INIT_SEQ_LEN; 2524 2525 auth_desc->fad_img_ae_insts_high = (uint64_t)bus_addr >> 32; 2526 auth_desc->fad_img_ae_insts_low = bus_addr; 2527 } else { 2528 auth_desc->fad_img_ae_insts_high = auth_desc->fad_img_high; 2529 auth_desc->fad_img_ae_insts_low = auth_desc->fad_img_low; 2530 } 2531 2532 bus_dmamap_sync(dma->qdm_dma_tag, dma->qdm_dma_map, 2533 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2534 2535 return 0; 2536} 2537 2538static int 2539qat_aefw_auth(struct qat_softc *sc, struct qat_dmamem *dma) 2540{ 2541 bus_addr_t addr; 2542 uint32_t fcu, sts; 2543 int retry = 0; 2544 2545 addr = dma->qdm_dma_seg.ds_addr; 2546 qat_cap_global_write_4(sc, FCU_DRAM_ADDR_HI, (uint64_t)addr >> 32); 2547 qat_cap_global_write_4(sc, FCU_DRAM_ADDR_LO, addr); 2548 qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_AUTH); 2549 2550 do { 2551 DELAY(FW_AUTH_WAIT_PERIOD * 1000); 2552 fcu = qat_cap_global_read_4(sc, FCU_STATUS); 2553 sts = __SHIFTOUT(fcu, FCU_STATUS_STS); 2554 if (sts == FCU_STATUS_STS_VERI_FAIL) 2555 goto fail; 2556 if (fcu & FCU_STATUS_AUTHFWLD && 2557 sts == FCU_STATUS_STS_VERI_DONE) { 2558 return 0; 2559 } 2560 } while (retry++ < FW_AUTH_MAX_RETRY); 2561 2562fail: 2563 device_printf(sc->sc_dev, 2564 "firmware authentication error: status 0x%08x retry %d\n", 2565 fcu, retry); 2566 return EINVAL; 2567} 2568 2569static int 2570qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma) 2571{ 2572 struct simg_ae_mode *ae_mode; 2573 uint32_t fcu, sts, loaded; 2574 u_int mask; 2575 u_char ae; 2576 int retry = 0; 2577 2578 ae_mode = (struct simg_ae_mode *)((uintptr_t)dma->qdm_dma_vaddr + 2579 sizeof(struct auth_chunk) + sizeof(struct css_hdr) + 2580 CSS_FWSK_PUB_LEN + CSS_SIGNATURE_LEN); 2581 2582 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 2583 if (!(mask & 1)) 2584 continue; 2585 if (!((ae_mode->sam_ae_mask >> ae) & 0x1)) 2586 continue; 2587 if (qat_ae_is_active(sc, ae)) { 2588 device_printf(sc->sc_dev, "AE %d is active\n", ae); 2589 return EINVAL; 2590 } 2591 qat_cap_global_write_4(sc, FCU_CTRL, 2592 FCU_CTRL_CMD_LOAD | __SHIFTIN(ae, FCU_CTRL_AE)); 2593 do { 2594 DELAY(FW_AUTH_WAIT_PERIOD * 1000); 2595 fcu = qat_cap_global_read_4(sc, FCU_STATUS); 2596 sts = __SHIFTOUT(fcu, FCU_STATUS_STS); 2597 loaded = __SHIFTOUT(fcu, FCU_STATUS_LOADED_AE); 2598 if (sts == FCU_STATUS_STS_LOAD_DONE && 2599 (loaded & (1 << ae))) { 2600 break; 2601 } 2602 } while (retry++ < FW_AUTH_MAX_RETRY); 2603 2604 if (retry > FW_AUTH_MAX_RETRY) { 2605 device_printf(sc->sc_dev, 2606 "firmware load timeout: status %08x\n", fcu); 2607 return EINVAL; 2608 } 2609 } 2610 2611 return 0; 2612} 2613 2614static int 2615qat_aefw_suof_write(struct qat_softc *sc) 2616{ 2617 struct qat_suof_image *qsi; 2618 int i, error = 0; 2619 2620 for (i = 0; i < sc->sc_aefw_suof.qafs_num_simgs; i++) { 2621 qsi = &sc->sc_aefw_suof.qafs_simg[i]; 2622 error = qat_aefw_alloc_auth_dmamem(sc, qsi->qsi_simg_buf, 2623 qsi->qsi_simg_len, &qsi->qsi_dma); 2624 if (error) 2625 return error; 2626 error = qat_aefw_auth(sc, &qsi->qsi_dma); 2627 if (error) { 2628 qat_free_dmamem(sc, &qsi->qsi_dma); 2629 return error; 2630 } 2631 error = qat_aefw_suof_load(sc, &qsi->qsi_dma); 2632 if (error) { 2633 qat_free_dmamem(sc, &qsi->qsi_dma); 2634 return error; 2635 } 2636 qat_free_dmamem(sc, &qsi->qsi_dma); 2637 } 2638 qat_free_mem(sc->sc_aefw_suof.qafs_simg); 2639 2640 return 0; 2641} 2642 2643static int 2644qat_aefw_uof_assign_image(struct qat_softc *sc, struct qat_ae *qae, 2645 struct qat_uof_image *qui) 2646{ 2647 struct qat_ae_slice *slice; 2648 int i, npages, nregions; 2649 2650 if (qae->qae_num_slices >= nitems(qae->qae_slices)) 2651 return ENOENT; 2652 2653 if (qui->qui_image->ui_ae_mode & 2654 (AE_MODE_RELOAD_CTX_SHARED | AE_MODE_SHARED_USTORE)) { 2655 /* XXX */ 2656 device_printf(sc->sc_dev, 2657 "shared ae mode is not supported yet\n"); 2658 return ENOTSUP; 2659 } 2660 2661 qae->qae_shareable_ustore = 0; /* XXX */ 2662 qae->qae_effect_ustore_size = USTORE_SIZE; 2663 2664 slice = &qae->qae_slices[qae->qae_num_slices]; 2665 2666 slice->qas_image = qui; 2667 slice->qas_assigned_ctx_mask = qui->qui_image->ui_ctx_assigned; 2668 2669 nregions = qui->qui_image->ui_num_page_regions; 2670 npages = qui->qui_image->ui_num_pages; 2671 2672 if (nregions > nitems(slice->qas_regions)) 2673 return ENOENT; 2674 if (npages > nitems(slice->qas_pages)) 2675 return ENOENT; 2676 2677 for (i = 0; i < nregions; i++) { 2678 STAILQ_INIT(&slice->qas_regions[i].qar_waiting_pages); 2679 } 2680 for (i = 0; i < npages; i++) { 2681 struct qat_ae_page *page = &slice->qas_pages[i]; 2682 int region; 2683 2684 page->qap_page = &qui->qui_pages[i]; 2685 region = page->qap_page->qup_page_region; 2686 if (region >= nregions) 2687 return EINVAL; 2688 2689 page->qap_region = &slice->qas_regions[region]; 2690 } 2691 2692 qae->qae_num_slices++; 2693 2694 return 0; 2695} 2696 2697static int 2698qat_aefw_uof_init_ae(struct qat_softc *sc, u_char ae) 2699{ 2700 struct uof_image *image; 2701 struct qat_ae *qae = &(QAT_AE(sc, ae)); 2702 int s; 2703 u_char nn_mode; 2704 2705 for (s = 0; s < qae->qae_num_slices; s++) { 2706 if (qae->qae_slices[s].qas_image == NULL) 2707 continue; 2708 2709 image = qae->qae_slices[s].qas_image->qui_image; 2710 qat_ae_write_ctx_mode(sc, ae, 2711 __SHIFTOUT(image->ui_ae_mode, AE_MODE_CTX_MODE)); 2712 2713 nn_mode = __SHIFTOUT(image->ui_ae_mode, AE_MODE_NN_MODE); 2714 if (nn_mode != AE_MODE_NN_MODE_DONTCARE) 2715 qat_ae_write_nn_mode(sc, ae, nn_mode); 2716 2717 qat_ae_write_lm_mode(sc, ae, AEREG_LMEM0, 2718 __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM0)); 2719 qat_ae_write_lm_mode(sc, ae, AEREG_LMEM1, 2720 __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM1)); 2721 2722 qat_ae_write_shared_cs_mode(sc, ae, 2723 __SHIFTOUT(image->ui_ae_mode, AE_MODE_SHARED_USTORE)); 2724 qat_ae_set_reload_ustore(sc, ae, image->ui_reloadable_size, 2725 __SHIFTOUT(image->ui_ae_mode, AE_MODE_RELOAD_CTX_SHARED), 2726 qae->qae_reloc_ustore_dram); 2727 } 2728 2729 return 0; 2730} 2731 2732static int 2733qat_aefw_uof_init(struct qat_softc *sc) 2734{ 2735 int ae, i, error; 2736 uint32_t mask; 2737 2738 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { 2739 struct qat_ae *qae; 2740 2741 if (!(mask & 1)) 2742 continue; 2743 2744 qae = &(QAT_AE(sc, ae)); 2745 2746 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { 2747 if ((sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned & 2748 (1 << ae)) == 0) 2749 continue; 2750 2751 error = qat_aefw_uof_assign_image(sc, qae, 2752 &sc->sc_aefw_uof.qafu_imgs[i]); 2753 if (error) 2754 return error; 2755 } 2756 2757 /* XXX UcLo_initNumUwordUsed */ 2758 2759 qae->qae_reloc_ustore_dram = UINT_MAX; /* XXX */ 2760 2761 error = qat_aefw_uof_init_ae(sc, ae); 2762 if (error) 2763 return error; 2764 } 2765 2766 return 0; 2767} 2768 2769int 2770qat_aefw_load(struct qat_softc *sc) 2771{ 2772 int error; 2773 2774 error = qat_aefw_load_mof(sc); 2775 if (error) 2776 return error; 2777 2778 error = qat_aefw_load_mmp(sc); 2779 if (error) 2780 return error; 2781 2782 error = qat_aefw_mof_parse(sc); 2783 if (error) { 2784 device_printf(sc->sc_dev, "couldn't parse mof: %d\n", error); 2785 return error; 2786 } 2787 2788 if (sc->sc_hw.qhw_fw_auth) { 2789 error = qat_aefw_suof_parse(sc); 2790 if (error) { 2791 device_printf(sc->sc_dev, "couldn't parse suof: %d\n", 2792 error); 2793 return error; 2794 } 2795 2796 error = qat_aefw_suof_write(sc); 2797 if (error) { 2798 device_printf(sc->sc_dev, 2799 "could not write firmware: %d\n", error); 2800 return error; 2801 } 2802 2803 } else { 2804 error = qat_aefw_uof_parse(sc); 2805 if (error) { 2806 device_printf(sc->sc_dev, "couldn't parse uof: %d\n", 2807 error); 2808 return error; 2809 } 2810 2811 error = qat_aefw_uof_init(sc); 2812 if (error) { 2813 device_printf(sc->sc_dev, 2814 "couldn't init for aefw: %d\n", error); 2815 return error; 2816 } 2817 2818 error = qat_aefw_uof_write(sc); 2819 if (error) { 2820 device_printf(sc->sc_dev, 2821 "Could not write firmware: %d\n", error); 2822 return error; 2823 } 2824 } 2825 2826 return 0; 2827} 2828 2829void 2830qat_aefw_unload(struct qat_softc *sc) 2831{ 2832 qat_aefw_unload_mmp(sc); 2833 qat_aefw_unload_mof(sc); 2834} 2835 2836int 2837qat_aefw_start(struct qat_softc *sc, u_char ae, u_int ctx_mask) 2838{ 2839 uint32_t fcu; 2840 int retry = 0; 2841 2842 if (sc->sc_hw.qhw_fw_auth) { 2843 qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_START); 2844 do { 2845 DELAY(FW_AUTH_WAIT_PERIOD * 1000); 2846 fcu = qat_cap_global_read_4(sc, FCU_STATUS); 2847 if (fcu & FCU_STATUS_DONE) 2848 return 0; 2849 } while (retry++ < FW_AUTH_MAX_RETRY); 2850 2851 device_printf(sc->sc_dev, 2852 "firmware start timeout: status %08x\n", fcu); 2853 return EINVAL; 2854 } else { 2855 qat_ae_ctx_indr_write(sc, ae, (~ctx_mask) & AE_ALL_CTX, 2856 CTX_WAKEUP_EVENTS_INDIRECT, 2857 CTX_WAKEUP_EVENTS_INDIRECT_SLEEP); 2858 qat_ae_enable_ctx(sc, ae, ctx_mask); 2859 } 2860 2861 return 0; 2862} 2863 2864static int 2865qat_aefw_init_memory_one(struct qat_softc *sc, struct uof_init_mem *uim) 2866{ 2867 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 2868 struct qat_ae_batch_init_list *qabi_list; 2869 struct uof_mem_val_attr *memattr; 2870 size_t *curinit; 2871 u_long ael; 2872 int i; 2873 const char *sym; 2874 char *ep; 2875 2876 memattr = (struct uof_mem_val_attr *)(uim + 1); 2877 2878 switch (uim->uim_region) { 2879 case LMEM_REGION: 2880 if ((uim->uim_addr + uim->uim_num_bytes) > MAX_LMEM_REG * 4) { 2881 device_printf(sc->sc_dev, 2882 "Invalid lmem addr or bytes\n"); 2883 return ENOBUFS; 2884 } 2885 if (uim->uim_scope != UOF_SCOPE_LOCAL) 2886 return EINVAL; 2887 sym = qat_aefw_uof_string(sc, uim->uim_sym_name); 2888 ael = strtoul(sym, &ep, 10); 2889 if (ep == sym || ael > MAX_AE) 2890 return EINVAL; 2891 if ((sc->sc_ae_mask & (1 << ael)) == 0) 2892 return 0; /* ae is fused out */ 2893 2894 curinit = &qafu->qafu_num_lm_init[ael]; 2895 qabi_list = &qafu->qafu_lm_init[ael]; 2896 2897 for (i = 0; i < uim->uim_num_val_attr; i++, memattr++) { 2898 struct qat_ae_batch_init *qabi; 2899 2900 qabi = qat_alloc_mem(sizeof(struct qat_ae_batch_init)); 2901 if (*curinit == 0) 2902 STAILQ_INIT(qabi_list); 2903 STAILQ_INSERT_TAIL(qabi_list, qabi, qabi_next); 2904 2905 qabi->qabi_ae = (u_int)ael; 2906 qabi->qabi_addr = 2907 uim->uim_addr + memattr->umva_byte_offset; 2908 qabi->qabi_value = &memattr->umva_value; 2909 qabi->qabi_size = 4; 2910 qafu->qafu_num_lm_init_inst[ael] += 2911 qat_ae_get_inst_num(qabi->qabi_size); 2912 (*curinit)++; 2913 if (*curinit >= MAX_LMEM_REG) { 2914 device_printf(sc->sc_dev, 2915 "Invalid lmem val attr\n"); 2916 return ENOBUFS; 2917 } 2918 } 2919 break; 2920 case SRAM_REGION: 2921 case DRAM_REGION: 2922 case DRAM1_REGION: 2923 case SCRATCH_REGION: 2924 case UMEM_REGION: 2925 /* XXX */ 2926 /* fallthrough */ 2927 default: 2928 device_printf(sc->sc_dev, 2929 "unsupported memory region to init: %d\n", 2930 uim->uim_region); 2931 return ENOTSUP; 2932 } 2933 2934 return 0; 2935} 2936 2937static void 2938qat_aefw_free_lm_init(struct qat_softc *sc, u_char ae) 2939{ 2940 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 2941 struct qat_ae_batch_init *qabi; 2942 2943 while ((qabi = STAILQ_FIRST(&qafu->qafu_lm_init[ae])) != NULL) { 2944 STAILQ_REMOVE_HEAD(&qafu->qafu_lm_init[ae], qabi_next); 2945 qat_free_mem(qabi); 2946 } 2947 2948 qafu->qafu_num_lm_init[ae] = 0; 2949 qafu->qafu_num_lm_init_inst[ae] = 0; 2950} 2951 2952static int 2953qat_aefw_init_ustore(struct qat_softc *sc) 2954{ 2955 uint64_t *fill; 2956 uint32_t dont_init; 2957 int a, i, p; 2958 int error = 0; 2959 int usz, end, start; 2960 u_char ae, nae; 2961 2962 fill = qat_alloc_mem(MAX_USTORE * sizeof(uint64_t)); 2963 2964 for (a = 0; a < sc->sc_aefw_uof.qafu_num_imgs; a++) { 2965 struct qat_uof_image *qui = &sc->sc_aefw_uof.qafu_imgs[a]; 2966 struct uof_image *ui = qui->qui_image; 2967 2968 for (i = 0; i < MAX_USTORE; i++) 2969 memcpy(&fill[i], ui->ui_fill_pattern, sizeof(uint64_t)); 2970 /* 2971 * Compute do_not_init value as a value that will not be equal 2972 * to fill data when cast to an int 2973 */ 2974 dont_init = 0; 2975 if (dont_init == (uint32_t)fill[0]) 2976 dont_init = 0xffffffff; 2977 2978 for (p = 0; p < ui->ui_num_pages; p++) { 2979 struct qat_uof_page *qup = &qui->qui_pages[p]; 2980 if (!qup->qup_def_page) 2981 continue; 2982 2983 for (i = qup->qup_beg_paddr; 2984 i < qup->qup_beg_paddr + qup->qup_num_micro_words; 2985 i++ ) { 2986 fill[i] = (uint64_t)dont_init; 2987 } 2988 } 2989 2990 for (ae = 0; ae < sc->sc_ae_num; ae++) { 2991 MPASS(ae < UOF_MAX_NUM_OF_AE); 2992 if ((ui->ui_ae_assigned & (1 << ae)) == 0) 2993 continue; 2994 2995 if (QAT_AE(sc, ae).qae_shareable_ustore && (ae & 1)) { 2996 qat_ae_get_shared_ustore_ae(ae, &nae); 2997 if (ui->ui_ae_assigned & (1 << ae)) 2998 continue; 2999 } 3000 usz = QAT_AE(sc, ae).qae_effect_ustore_size; 3001 3002 /* initialize the areas not going to be overwritten */ 3003 end = -1; 3004 do { 3005 /* find next uword that needs to be initialized */ 3006 for (start = end + 1; start < usz; start++) { 3007 if ((uint32_t)fill[start] != dont_init) 3008 break; 3009 } 3010 /* see if there are no more such uwords */ 3011 if (start >= usz) 3012 break; 3013 for (end = start + 1; end < usz; end++) { 3014 if ((uint32_t)fill[end] == dont_init) 3015 break; 3016 } 3017 if (QAT_AE(sc, ae).qae_shareable_ustore) { 3018 error = ENOTSUP; /* XXX */ 3019 goto out; 3020 } else { 3021 error = qat_ae_ucode_write(sc, ae, 3022 start, end - start, &fill[start]); 3023 if (error) { 3024 goto out; 3025 } 3026 } 3027 3028 } while (end < usz); 3029 } 3030 } 3031 3032out: 3033 qat_free_mem(fill); 3034 return error; 3035} 3036 3037static int 3038qat_aefw_init_reg(struct qat_softc *sc, u_char ae, u_char ctx_mask, 3039 enum aereg_type regtype, u_short regaddr, u_int value) 3040{ 3041 int error = 0; 3042 u_char ctx; 3043 3044 switch (regtype) { 3045 case AEREG_GPA_REL: 3046 case AEREG_GPB_REL: 3047 case AEREG_SR_REL: 3048 case AEREG_SR_RD_REL: 3049 case AEREG_SR_WR_REL: 3050 case AEREG_DR_REL: 3051 case AEREG_DR_RD_REL: 3052 case AEREG_DR_WR_REL: 3053 case AEREG_NEIGH_REL: 3054 /* init for all valid ctx */ 3055 for (ctx = 0; ctx < MAX_AE_CTX; ctx++) { 3056 if ((ctx_mask & (1 << ctx)) == 0) 3057 continue; 3058 error = qat_aereg_rel_data_write(sc, ae, ctx, regtype, 3059 regaddr, value); 3060 } 3061 break; 3062 case AEREG_GPA_ABS: 3063 case AEREG_GPB_ABS: 3064 case AEREG_SR_ABS: 3065 case AEREG_SR_RD_ABS: 3066 case AEREG_SR_WR_ABS: 3067 case AEREG_DR_ABS: 3068 case AEREG_DR_RD_ABS: 3069 case AEREG_DR_WR_ABS: 3070 error = qat_aereg_abs_data_write(sc, ae, regtype, 3071 regaddr, value); 3072 break; 3073 default: 3074 error = EINVAL; 3075 break; 3076 } 3077 3078 return error; 3079} 3080 3081static int 3082qat_aefw_init_reg_sym_expr(struct qat_softc *sc, u_char ae, 3083 struct qat_uof_image *qui) 3084{ 3085 u_int i, expres; 3086 u_char ctx_mask; 3087 3088 for (i = 0; i < qui->qui_num_init_reg_sym; i++) { 3089 struct uof_init_reg_sym *uirs = &qui->qui_init_reg_sym[i]; 3090 3091 if (uirs->uirs_value_type == EXPR_VAL) { 3092 /* XXX */ 3093 device_printf(sc->sc_dev, 3094 "does not support initializing EXPR_VAL\n"); 3095 return ENOTSUP; 3096 } else { 3097 expres = uirs->uirs_value; 3098 } 3099 3100 switch (uirs->uirs_init_type) { 3101 case INIT_REG: 3102 if (__SHIFTOUT(qui->qui_image->ui_ae_mode, 3103 AE_MODE_CTX_MODE) == MAX_AE_CTX) { 3104 ctx_mask = 0xff; /* 8-ctx mode */ 3105 } else { 3106 ctx_mask = 0x55; /* 4-ctx mode */ 3107 } 3108 qat_aefw_init_reg(sc, ae, ctx_mask, 3109 (enum aereg_type)uirs->uirs_reg_type, 3110 (u_short)uirs->uirs_addr_offset, expres); 3111 break; 3112 case INIT_REG_CTX: 3113 if (__SHIFTOUT(qui->qui_image->ui_ae_mode, 3114 AE_MODE_CTX_MODE) == MAX_AE_CTX) { 3115 ctx_mask = 0xff; /* 8-ctx mode */ 3116 } else { 3117 ctx_mask = 0x55; /* 4-ctx mode */ 3118 } 3119 if (((1 << uirs->uirs_ctx) & ctx_mask) == 0) 3120 return EINVAL; 3121 qat_aefw_init_reg(sc, ae, 1 << uirs->uirs_ctx, 3122 (enum aereg_type)uirs->uirs_reg_type, 3123 (u_short)uirs->uirs_addr_offset, expres); 3124 break; 3125 case INIT_EXPR: 3126 case INIT_EXPR_ENDIAN_SWAP: 3127 default: 3128 device_printf(sc->sc_dev, 3129 "does not support initializing init_type %d\n", 3130 uirs->uirs_init_type); 3131 return ENOTSUP; 3132 } 3133 } 3134 3135 return 0; 3136} 3137 3138static int 3139qat_aefw_init_memory(struct qat_softc *sc) 3140{ 3141 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 3142 size_t uimsz, initmemsz = qafu->qafu_init_mem_size; 3143 struct uof_init_mem *uim; 3144 int error, i; 3145 u_char ae; 3146 3147 uim = qafu->qafu_init_mem; 3148 for (i = 0; i < qafu->qafu_num_init_mem; i++) { 3149 uimsz = sizeof(struct uof_init_mem) + 3150 sizeof(struct uof_mem_val_attr) * uim->uim_num_val_attr; 3151 if (uimsz > initmemsz) { 3152 device_printf(sc->sc_dev, 3153 "invalid uof_init_mem or uof_mem_val_attr size\n"); 3154 return EINVAL; 3155 } 3156 3157 if (uim->uim_num_bytes > 0) { 3158 error = qat_aefw_init_memory_one(sc, uim); 3159 if (error) { 3160 device_printf(sc->sc_dev, 3161 "Could not init ae memory: %d\n", error); 3162 return error; 3163 } 3164 } 3165 uim = (struct uof_init_mem *)((uintptr_t)uim + uimsz); 3166 initmemsz -= uimsz; 3167 } 3168 3169 /* run Batch put LM API */ 3170 for (ae = 0; ae < MAX_AE; ae++) { 3171 error = qat_ae_batch_put_lm(sc, ae, &qafu->qafu_lm_init[ae], 3172 qafu->qafu_num_lm_init_inst[ae]); 3173 if (error) 3174 device_printf(sc->sc_dev, "Could not put lm\n"); 3175 3176 qat_aefw_free_lm_init(sc, ae); 3177 } 3178 3179 error = qat_aefw_init_ustore(sc); 3180 3181 /* XXX run Batch put LM API */ 3182 3183 return error; 3184} 3185 3186static int 3187qat_aefw_init_globals(struct qat_softc *sc) 3188{ 3189 struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; 3190 int error, i, p, s; 3191 u_char ae; 3192 3193 /* initialize the memory segments */ 3194 if (qafu->qafu_num_init_mem > 0) { 3195 error = qat_aefw_init_memory(sc); 3196 if (error) 3197 return error; 3198 } else { 3199 error = qat_aefw_init_ustore(sc); 3200 if (error) 3201 return error; 3202 } 3203 3204 /* XXX bind import variables with ivd values */ 3205 3206 /* XXX bind the uC global variables 3207 * local variables will done on-the-fly */ 3208 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { 3209 for (p = 0; p < sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_num_pages; p++) { 3210 struct qat_uof_page *qup = 3211 &sc->sc_aefw_uof.qafu_imgs[i].qui_pages[p]; 3212 if (qup->qup_num_uw_blocks && 3213 (qup->qup_num_uc_var || qup->qup_num_imp_var)) { 3214 device_printf(sc->sc_dev, 3215 "not support uC global variables\n"); 3216 return ENOTSUP; 3217 } 3218 } 3219 } 3220 3221 for (ae = 0; ae < sc->sc_ae_num; ae++) { 3222 struct qat_ae *qae = &(QAT_AE(sc, ae)); 3223 3224 for (s = 0; s < qae->qae_num_slices; s++) { 3225 struct qat_ae_slice *qas = &qae->qae_slices[s]; 3226 3227 if (qas->qas_image == NULL) 3228 continue; 3229 3230 error = 3231 qat_aefw_init_reg_sym_expr(sc, ae, qas->qas_image); 3232 if (error) 3233 return error; 3234 } 3235 } 3236 3237 return 0; 3238} 3239 3240static uint64_t 3241qat_aefw_get_uof_inst(struct qat_softc *sc, struct qat_uof_page *qup, 3242 u_int addr) 3243{ 3244 uint64_t uinst = 0; 3245 u_int i; 3246 3247 /* find the block */ 3248 for (i = 0; i < qup->qup_num_uw_blocks; i++) { 3249 struct qat_uof_uword_block *quub = &qup->qup_uw_blocks[i]; 3250 3251 if ((addr >= quub->quub_start_addr) && 3252 (addr <= (quub->quub_start_addr + 3253 (quub->quub_num_words - 1)))) { 3254 /* unpack n bytes and assigned to the 64-bit uword value. 3255 note: the microwords are stored as packed bytes. 3256 */ 3257 addr -= quub->quub_start_addr; 3258 addr *= AEV2_PACKED_UWORD_BYTES; 3259 memcpy(&uinst, 3260 (void *)((uintptr_t)quub->quub_micro_words + addr), 3261 AEV2_PACKED_UWORD_BYTES); 3262 uinst = uinst & UWORD_MASK; 3263 3264 return uinst; 3265 } 3266 } 3267 3268 return INVLD_UWORD; 3269} 3270 3271static int 3272qat_aefw_do_pagein(struct qat_softc *sc, u_char ae, struct qat_uof_page *qup) 3273{ 3274 struct qat_ae *qae = &(QAT_AE(sc, ae)); 3275 uint64_t fill, *ucode_cpybuf; 3276 u_int error, i, upaddr, uraddr, ninst, cpylen; 3277 3278 if (qup->qup_num_uc_var || qup->qup_num_neigh_reg || 3279 qup->qup_num_imp_var || qup->qup_num_imp_expr) { 3280 device_printf(sc->sc_dev, 3281 "does not support fixup locals\n"); 3282 return ENOTSUP; 3283 } 3284 3285 ucode_cpybuf = qat_alloc_mem(UWORD_CPYBUF_SIZE * sizeof(uint64_t)); 3286 3287 /* XXX get fill-pattern from an image -- they are all the same */ 3288 memcpy(&fill, sc->sc_aefw_uof.qafu_imgs[0].qui_image->ui_fill_pattern, 3289 sizeof(uint64_t)); 3290 3291 upaddr = qup->qup_beg_paddr; 3292 uraddr = 0; 3293 ninst = qup->qup_num_micro_words; 3294 while (ninst > 0) { 3295 cpylen = min(ninst, UWORD_CPYBUF_SIZE); 3296 3297 /* load the buffer */ 3298 for (i = 0; i < cpylen; i++) { 3299 /* keep below code structure in case there are 3300 * different handling for shared secnarios */ 3301 if (!qae->qae_shareable_ustore) { 3302 /* qat_aefw_get_uof_inst() takes an address that 3303 * is relative to the start of the page. 3304 * So we don't need to add in the physical 3305 * offset of the page. */ 3306 if (qup->qup_page_region != 0) { 3307 /* XXX */ 3308 device_printf(sc->sc_dev, 3309 "region != 0 is not supported\n"); 3310 qat_free_mem(ucode_cpybuf); 3311 return ENOTSUP; 3312 } else { 3313 /* for mixing case, it should take 3314 * physical address */ 3315 ucode_cpybuf[i] = qat_aefw_get_uof_inst( 3316 sc, qup, upaddr + i); 3317 if (ucode_cpybuf[i] == INVLD_UWORD) { 3318 /* fill hole in the uof */ 3319 ucode_cpybuf[i] = fill; 3320 } 3321 } 3322 } else { 3323 /* XXX */ 3324 qat_free_mem(ucode_cpybuf); 3325 return ENOTSUP; 3326 } 3327 } 3328 3329 /* copy the buffer to ustore */ 3330 if (!qae->qae_shareable_ustore) { 3331 error = qat_ae_ucode_write(sc, ae, upaddr, cpylen, 3332 ucode_cpybuf); 3333 if (error) 3334 return error; 3335 } else { 3336 /* XXX */ 3337 qat_free_mem(ucode_cpybuf); 3338 return ENOTSUP; 3339 } 3340 upaddr += cpylen; 3341 uraddr += cpylen; 3342 ninst -= cpylen; 3343 } 3344 3345 qat_free_mem(ucode_cpybuf); 3346 3347 return 0; 3348} 3349 3350static int 3351qat_aefw_uof_write_one(struct qat_softc *sc, struct qat_uof_image *qui) 3352{ 3353 struct uof_image *ui = qui->qui_image; 3354 struct qat_ae_page *qap; 3355 u_int s, p, c; 3356 int error; 3357 u_char ae, ctx_mask; 3358 3359 if (__SHIFTOUT(ui->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX) 3360 ctx_mask = 0xff; /* 8-ctx mode */ 3361 else 3362 ctx_mask = 0x55; /* 4-ctx mode */ 3363 3364 /* load the default page and set assigned CTX PC 3365 * to the entrypoint address */ 3366 for (ae = 0; ae < sc->sc_ae_num; ae++) { 3367 struct qat_ae *qae = &(QAT_AE(sc, ae)); 3368 struct qat_ae_slice *qas; 3369 u_int metadata; 3370 3371 MPASS(ae < UOF_MAX_NUM_OF_AE); 3372 3373 if ((ui->ui_ae_assigned & (1 << ae)) == 0) 3374 continue; 3375 3376 /* find the slice to which this image is assigned */ 3377 for (s = 0; s < qae->qae_num_slices; s++) { 3378 qas = &qae->qae_slices[s]; 3379 if (ui->ui_ctx_assigned & qas->qas_assigned_ctx_mask) 3380 break; 3381 } 3382 if (s >= qae->qae_num_slices) 3383 continue; 3384 3385 qas = &qae->qae_slices[s]; 3386 3387 for (p = 0; p < ui->ui_num_pages; p++) { 3388 qap = &qas->qas_pages[p]; 3389 3390 /* Only load pages loaded by default */ 3391 if (!qap->qap_page->qup_def_page) 3392 continue; 3393 3394 error = qat_aefw_do_pagein(sc, ae, qap->qap_page); 3395 if (error) 3396 return error; 3397 } 3398 3399 metadata = qas->qas_image->qui_image->ui_app_metadata; 3400 if (metadata != 0xffffffff && bootverbose) { 3401 device_printf(sc->sc_dev, 3402 "loaded firmware: %s\n", 3403 qat_aefw_uof_string(sc, metadata)); 3404 } 3405 3406 /* Assume starting page is page 0 */ 3407 qap = &qas->qas_pages[0]; 3408 for (c = 0; c < MAX_AE_CTX; c++) { 3409 if (ctx_mask & (1 << c)) 3410 qas->qas_cur_pages[c] = qap; 3411 else 3412 qas->qas_cur_pages[c] = NULL; 3413 } 3414 3415 /* set the live context */ 3416 qae->qae_live_ctx_mask = ui->ui_ctx_assigned; 3417 3418 /* set context PC to the image entrypoint address */ 3419 error = qat_ae_write_pc(sc, ae, ui->ui_ctx_assigned, 3420 ui->ui_entry_address); 3421 if (error) 3422 return error; 3423 } 3424 3425 /* XXX store the checksum for convenience */ 3426 3427 return 0; 3428} 3429 3430static int 3431qat_aefw_uof_write(struct qat_softc *sc) 3432{ 3433 int error = 0; 3434 int i; 3435 3436 error = qat_aefw_init_globals(sc); 3437 if (error) { 3438 device_printf(sc->sc_dev, 3439 "Could not initialize globals\n"); 3440 return error; 3441 } 3442 3443 for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { 3444 error = qat_aefw_uof_write_one(sc, 3445 &sc->sc_aefw_uof.qafu_imgs[i]); 3446 if (error) 3447 break; 3448 } 3449 3450 /* XXX UcLo_computeFreeUstore */ 3451 3452 return error; 3453} 3454