1// SPDX-License-Identifier: Intel 2/* 3 * Copyright (C) 2013, Intel Corporation 4 * Copyright (C) 2015, Bin Meng <bmeng.cn@gmail.com> 5 * 6 * Ported from Intel released Quark UEFI BIOS 7 * QuarkSocPkg/QuarkNorthCluster/MemoryInit/Pei 8 */ 9 10#include <common.h> 11#include <pci.h> 12#include <asm/arch/device.h> 13#include <asm/arch/mrc.h> 14#include <asm/arch/msg_port.h> 15#include "mrc_util.h" 16#include "hte.h" 17#include "smc.h" 18 19/* t_ck clock period in picoseconds per speed index 800, 1066, 1333 */ 20static const uint32_t t_ck[3] = { 21 2500, 22 1875, 23 1500 24}; 25 26/* Global variables */ 27static const uint16_t ddr_wclk[] = {193, 158}; 28#ifdef BACKUP_WCTL 29static const uint16_t ddr_wctl[] = {1, 217}; 30#endif 31#ifdef BACKUP_WCMD 32static const uint16_t ddr_wcmd[] = {1, 220}; 33#endif 34 35#ifdef BACKUP_RCVN 36static const uint16_t ddr_rcvn[] = {129, 498}; 37#endif 38 39#ifdef BACKUP_WDQS 40static const uint16_t ddr_wdqs[] = {65, 289}; 41#endif 42 43#ifdef BACKUP_RDQS 44static const uint8_t ddr_rdqs[] = {32, 24}; 45#endif 46 47#ifdef BACKUP_WDQ 48static const uint16_t ddr_wdq[] = {32, 257}; 49#endif 50 51/* Stop self refresh driven by MCU */ 52void clear_self_refresh(struct mrc_params *mrc_params) 53{ 54 ENTERFN(); 55 56 /* clear the PMSTS Channel Self Refresh bits */ 57 mrc_write_mask(MEM_CTLR, PMSTS, PMSTS_DISR, PMSTS_DISR); 58 59 LEAVEFN(); 60} 61 62/* It will initialize timing registers in the MCU (DTR0..DTR4) */ 63void prog_ddr_timing_control(struct mrc_params *mrc_params) 64{ 65 uint8_t tcl, wl; 66 uint8_t trp, trcd, tras, twr, twtr, trrd, trtp, tfaw; 67 uint32_t tck; 68 u32 dtr0, dtr1, dtr2, dtr3, dtr4; 69 u32 tmp1, tmp2; 70 71 ENTERFN(); 72 73 /* mcu_init starts */ 74 mrc_post_code(0x02, 0x00); 75 76 dtr0 = msg_port_read(MEM_CTLR, DTR0); 77 dtr1 = msg_port_read(MEM_CTLR, DTR1); 78 dtr2 = msg_port_read(MEM_CTLR, DTR2); 79 dtr3 = msg_port_read(MEM_CTLR, DTR3); 80 dtr4 = msg_port_read(MEM_CTLR, DTR4); 81 82 tck = t_ck[mrc_params->ddr_speed]; /* Clock in picoseconds */ 83 tcl = mrc_params->params.cl; /* CAS latency in clocks */ 84 trp = tcl; /* Per CAT MRC */ 85 trcd = tcl; /* Per CAT MRC */ 86 tras = MCEIL(mrc_params->params.ras, tck); 87 88 /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */ 89 twr = MCEIL(15000, tck); 90 91 twtr = MCEIL(mrc_params->params.wtr, tck); 92 trrd = MCEIL(mrc_params->params.rrd, tck); 93 trtp = 4; /* Valid for 800 and 1066, use 5 for 1333 */ 94 tfaw = MCEIL(mrc_params->params.faw, tck); 95 96 wl = 5 + mrc_params->ddr_speed; 97 98 dtr0 &= ~DTR0_DFREQ_MASK; 99 dtr0 |= mrc_params->ddr_speed; 100 dtr0 &= ~DTR0_TCL_MASK; 101 tmp1 = tcl - 5; 102 dtr0 |= ((tcl - 5) << 12); 103 dtr0 &= ~DTR0_TRP_MASK; 104 dtr0 |= ((trp - 5) << 4); /* 5 bit DRAM Clock */ 105 dtr0 &= ~DTR0_TRCD_MASK; 106 dtr0 |= ((trcd - 5) << 8); /* 5 bit DRAM Clock */ 107 108 dtr1 &= ~DTR1_TWCL_MASK; 109 tmp2 = wl - 3; 110 dtr1 |= (wl - 3); 111 dtr1 &= ~DTR1_TWTP_MASK; 112 dtr1 |= ((wl + 4 + twr - 14) << 8); /* Change to tWTP */ 113 dtr1 &= ~DTR1_TRTP_MASK; 114 dtr1 |= ((MMAX(trtp, 4) - 3) << 28); /* 4 bit DRAM Clock */ 115 dtr1 &= ~DTR1_TRRD_MASK; 116 dtr1 |= ((trrd - 4) << 24); /* 4 bit DRAM Clock */ 117 dtr1 &= ~DTR1_TCMD_MASK; 118 dtr1 |= (1 << 4); 119 dtr1 &= ~DTR1_TRAS_MASK; 120 dtr1 |= ((tras - 14) << 20); /* 6 bit DRAM Clock */ 121 dtr1 &= ~DTR1_TFAW_MASK; 122 dtr1 |= ((((tfaw + 1) >> 1) - 5) << 16);/* 4 bit DRAM Clock */ 123 /* Set 4 Clock CAS to CAS delay (multi-burst) */ 124 dtr1 &= ~DTR1_TCCD_MASK; 125 126 dtr2 &= ~DTR2_TRRDR_MASK; 127 dtr2 |= 1; 128 dtr2 &= ~DTR2_TWWDR_MASK; 129 dtr2 |= (2 << 8); 130 dtr2 &= ~DTR2_TRWDR_MASK; 131 dtr2 |= (2 << 16); 132 133 dtr3 &= ~DTR3_TWRDR_MASK; 134 dtr3 |= 2; 135 dtr3 &= ~DTR3_TXXXX_MASK; 136 dtr3 |= (2 << 4); 137 138 dtr3 &= ~DTR3_TRWSR_MASK; 139 if (mrc_params->ddr_speed == DDRFREQ_800) { 140 /* Extended RW delay (+1) */ 141 dtr3 |= ((tcl - 5 + 1) << 8); 142 } else if (mrc_params->ddr_speed == DDRFREQ_1066) { 143 /* Extended RW delay (+1) */ 144 dtr3 |= ((tcl - 5 + 1) << 8); 145 } 146 147 dtr3 &= ~DTR3_TWRSR_MASK; 148 dtr3 |= ((4 + wl + twtr - 11) << 13); 149 150 dtr3 &= ~DTR3_TXP_MASK; 151 if (mrc_params->ddr_speed == DDRFREQ_800) 152 dtr3 |= ((MMAX(0, 1 - 1)) << 22); 153 else 154 dtr3 |= ((MMAX(0, 2 - 1)) << 22); 155 156 dtr4 &= ~DTR4_WRODTSTRT_MASK; 157 dtr4 |= 1; 158 dtr4 &= ~DTR4_WRODTSTOP_MASK; 159 dtr4 |= (1 << 4); 160 dtr4 &= ~DTR4_XXXX1_MASK; 161 dtr4 |= ((1 + tmp1 - tmp2 + 2) << 8); 162 dtr4 &= ~DTR4_XXXX2_MASK; 163 dtr4 |= ((1 + tmp1 - tmp2 + 2) << 12); 164 dtr4 &= ~(DTR4_ODTDIS | DTR4_TRGSTRDIS); 165 166 msg_port_write(MEM_CTLR, DTR0, dtr0); 167 msg_port_write(MEM_CTLR, DTR1, dtr1); 168 msg_port_write(MEM_CTLR, DTR2, dtr2); 169 msg_port_write(MEM_CTLR, DTR3, dtr3); 170 msg_port_write(MEM_CTLR, DTR4, dtr4); 171 172 LEAVEFN(); 173} 174 175/* Configure MCU before jedec init sequence */ 176void prog_decode_before_jedec(struct mrc_params *mrc_params) 177{ 178 u32 drp; 179 u32 drfc; 180 u32 dcal; 181 u32 dsch; 182 u32 dpmc0; 183 184 ENTERFN(); 185 186 /* Disable power saving features */ 187 dpmc0 = msg_port_read(MEM_CTLR, DPMC0); 188 dpmc0 |= (DPMC0_CLKGTDIS | DPMC0_DISPWRDN); 189 dpmc0 &= ~DPMC0_PCLSTO_MASK; 190 dpmc0 &= ~DPMC0_DYNSREN; 191 msg_port_write(MEM_CTLR, DPMC0, dpmc0); 192 193 /* Disable out of order transactions */ 194 dsch = msg_port_read(MEM_CTLR, DSCH); 195 dsch |= (DSCH_OOODIS | DSCH_NEWBYPDIS); 196 msg_port_write(MEM_CTLR, DSCH, dsch); 197 198 /* Disable issuing the REF command */ 199 drfc = msg_port_read(MEM_CTLR, DRFC); 200 drfc &= ~DRFC_TREFI_MASK; 201 msg_port_write(MEM_CTLR, DRFC, drfc); 202 203 /* Disable ZQ calibration short */ 204 dcal = msg_port_read(MEM_CTLR, DCAL); 205 dcal &= ~DCAL_ZQCINT_MASK; 206 dcal &= ~DCAL_SRXZQCL_MASK; 207 msg_port_write(MEM_CTLR, DCAL, dcal); 208 209 /* 210 * Training performed in address mode 0, rank population has limited 211 * impact, however simulator complains if enabled non-existing rank. 212 */ 213 drp = 0; 214 if (mrc_params->rank_enables & 1) 215 drp |= DRP_RKEN0; 216 if (mrc_params->rank_enables & 2) 217 drp |= DRP_RKEN1; 218 msg_port_write(MEM_CTLR, DRP, drp); 219 220 LEAVEFN(); 221} 222 223/* 224 * After Cold Reset, BIOS should set COLDWAKE bit to 1 before 225 * sending the WAKE message to the Dunit. 226 * 227 * For Standby Exit, or any other mode in which the DRAM is in 228 * SR, this bit must be set to 0. 229 */ 230void perform_ddr_reset(struct mrc_params *mrc_params) 231{ 232 ENTERFN(); 233 234 /* Set COLDWAKE bit before sending the WAKE message */ 235 mrc_write_mask(MEM_CTLR, DRMC, DRMC_COLDWAKE, DRMC_COLDWAKE); 236 237 /* Send wake command to DUNIT (MUST be done before JEDEC) */ 238 dram_wake_command(); 239 240 /* Set default value */ 241 msg_port_write(MEM_CTLR, DRMC, 242 mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0); 243 244 LEAVEFN(); 245} 246 247 248/* 249 * This function performs some initialization on the DDRIO unit. 250 * This function is dependent on BOARD_ID, DDR_SPEED, and CHANNEL_ENABLES. 251 */ 252void ddrphy_init(struct mrc_params *mrc_params) 253{ 254 uint32_t temp; 255 uint8_t ch; /* channel counter */ 256 uint8_t rk; /* rank counter */ 257 uint8_t bl_grp; /* byte lane group counter (2 BLs per module) */ 258 uint8_t bl_divisor = 1; /* byte lane divisor */ 259 /* For DDR3 --> 0 == 800, 1 == 1066, 2 == 1333 */ 260 uint8_t speed = mrc_params->ddr_speed & 3; 261 uint8_t cas; 262 uint8_t cwl; 263 264 ENTERFN(); 265 266 cas = mrc_params->params.cl; 267 cwl = 5 + mrc_params->ddr_speed; 268 269 /* ddrphy_init starts */ 270 mrc_post_code(0x03, 0x00); 271 272 /* 273 * HSD#231531 274 * Make sure IOBUFACT is deasserted before initializing the DDR PHY 275 * 276 * HSD#234845 277 * Make sure WRPTRENABLE is deasserted before initializing the DDR PHY 278 */ 279 for (ch = 0; ch < NUM_CHANNELS; ch++) { 280 if (mrc_params->channel_enables & (1 << ch)) { 281 /* Deassert DDRPHY Initialization Complete */ 282 mrc_alt_write_mask(DDRPHY, 283 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET, 284 ~(1 << 20), 1 << 20); /* SPID_INIT_COMPLETE=0 */ 285 /* Deassert IOBUFACT */ 286 mrc_alt_write_mask(DDRPHY, 287 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET, 288 ~(1 << 2), 1 << 2); /* IOBUFACTRST_N=0 */ 289 /* Disable WRPTR */ 290 mrc_alt_write_mask(DDRPHY, 291 CMDPTRREG + ch * DDRIOCCC_CH_OFFSET, 292 ~(1 << 0), 1 << 0); /* WRPTRENABLE=0 */ 293 } 294 } 295 296 /* Put PHY in reset */ 297 mrc_alt_write_mask(DDRPHY, MASTERRSTN, 0, 1); 298 299 /* Initialize DQ01, DQ23, CMD, CLK-CTL, COMP modules */ 300 301 /* STEP0 */ 302 mrc_post_code(0x03, 0x10); 303 for (ch = 0; ch < NUM_CHANNELS; ch++) { 304 if (mrc_params->channel_enables & (1 << ch)) { 305 /* DQ01-DQ23 */ 306 for (bl_grp = 0; 307 bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2; 308 bl_grp++) { 309 /* Analog MUX select - IO2xCLKSEL */ 310 mrc_alt_write_mask(DDRPHY, 311 DQOBSCKEBBCTL + 312 bl_grp * DDRIODQ_BL_OFFSET + 313 ch * DDRIODQ_CH_OFFSET, 314 bl_grp ? 0 : (1 << 22), 1 << 22); 315 316 /* ODT Strength */ 317 switch (mrc_params->rd_odt_value) { 318 case 1: 319 temp = 0x3; 320 break; /* 60 ohm */ 321 case 2: 322 temp = 0x3; 323 break; /* 120 ohm */ 324 case 3: 325 temp = 0x3; 326 break; /* 180 ohm */ 327 default: 328 temp = 0x3; 329 break; /* 120 ohm */ 330 } 331 332 /* ODT strength */ 333 mrc_alt_write_mask(DDRPHY, 334 B0RXIOBUFCTL + 335 bl_grp * DDRIODQ_BL_OFFSET + 336 ch * DDRIODQ_CH_OFFSET, 337 temp << 5, 0x60); 338 /* ODT strength */ 339 mrc_alt_write_mask(DDRPHY, 340 B1RXIOBUFCTL + 341 bl_grp * DDRIODQ_BL_OFFSET + 342 ch * DDRIODQ_CH_OFFSET, 343 temp << 5, 0x60); 344 345 /* Dynamic ODT/DIFFAMP */ 346 temp = (cas << 24) | (cas << 16) | 347 (cas << 8) | (cas << 0); 348 switch (speed) { 349 case 0: 350 temp -= 0x01010101; 351 break; /* 800 */ 352 case 1: 353 temp -= 0x02020202; 354 break; /* 1066 */ 355 case 2: 356 temp -= 0x03030303; 357 break; /* 1333 */ 358 case 3: 359 temp -= 0x04040404; 360 break; /* 1600 */ 361 } 362 363 /* Launch Time: ODT, DIFFAMP, ODT, DIFFAMP */ 364 mrc_alt_write_mask(DDRPHY, 365 B01LATCTL1 + 366 bl_grp * DDRIODQ_BL_OFFSET + 367 ch * DDRIODQ_CH_OFFSET, 368 temp, 0x1f1f1f1f); 369 switch (speed) { 370 /* HSD#234715 */ 371 case 0: 372 temp = (0x06 << 16) | (0x07 << 8); 373 break; /* 800 */ 374 case 1: 375 temp = (0x07 << 16) | (0x08 << 8); 376 break; /* 1066 */ 377 case 2: 378 temp = (0x09 << 16) | (0x0a << 8); 379 break; /* 1333 */ 380 case 3: 381 temp = (0x0a << 16) | (0x0b << 8); 382 break; /* 1600 */ 383 } 384 385 /* On Duration: ODT, DIFFAMP */ 386 mrc_alt_write_mask(DDRPHY, 387 B0ONDURCTL + 388 bl_grp * DDRIODQ_BL_OFFSET + 389 ch * DDRIODQ_CH_OFFSET, 390 temp, 0x003f3f00); 391 /* On Duration: ODT, DIFFAMP */ 392 mrc_alt_write_mask(DDRPHY, 393 B1ONDURCTL + 394 bl_grp * DDRIODQ_BL_OFFSET + 395 ch * DDRIODQ_CH_OFFSET, 396 temp, 0x003f3f00); 397 398 switch (mrc_params->rd_odt_value) { 399 case 0: 400 /* override DIFFAMP=on, ODT=off */ 401 temp = (0x3f << 16) | (0x3f << 10); 402 break; 403 default: 404 /* override DIFFAMP=on, ODT=on */ 405 temp = (0x3f << 16) | (0x2a << 10); 406 break; 407 } 408 409 /* Override: DIFFAMP, ODT */ 410 mrc_alt_write_mask(DDRPHY, 411 B0OVRCTL + 412 bl_grp * DDRIODQ_BL_OFFSET + 413 ch * DDRIODQ_CH_OFFSET, 414 temp, 0x003ffc00); 415 /* Override: DIFFAMP, ODT */ 416 mrc_alt_write_mask(DDRPHY, 417 B1OVRCTL + 418 bl_grp * DDRIODQ_BL_OFFSET + 419 ch * DDRIODQ_CH_OFFSET, 420 temp, 0x003ffc00); 421 422 /* DLL Setup */ 423 424 /* 1xCLK Domain Timings: tEDP,RCVEN,WDQS (PO) */ 425 mrc_alt_write_mask(DDRPHY, 426 B0LATCTL0 + 427 bl_grp * DDRIODQ_BL_OFFSET + 428 ch * DDRIODQ_CH_OFFSET, 429 ((cas + 7) << 16) | ((cas - 4) << 8) | 430 ((cwl - 2) << 0), 0x003f1f1f); 431 mrc_alt_write_mask(DDRPHY, 432 B1LATCTL0 + 433 bl_grp * DDRIODQ_BL_OFFSET + 434 ch * DDRIODQ_CH_OFFSET, 435 ((cas + 7) << 16) | ((cas - 4) << 8) | 436 ((cwl - 2) << 0), 0x003f1f1f); 437 438 /* RCVEN Bypass (PO) */ 439 mrc_alt_write_mask(DDRPHY, 440 B0RXIOBUFCTL + 441 bl_grp * DDRIODQ_BL_OFFSET + 442 ch * DDRIODQ_CH_OFFSET, 443 0, 0x81); 444 mrc_alt_write_mask(DDRPHY, 445 B1RXIOBUFCTL + 446 bl_grp * DDRIODQ_BL_OFFSET + 447 ch * DDRIODQ_CH_OFFSET, 448 0, 0x81); 449 450 /* TX */ 451 mrc_alt_write_mask(DDRPHY, 452 DQCTL + 453 bl_grp * DDRIODQ_BL_OFFSET + 454 ch * DDRIODQ_CH_OFFSET, 455 1 << 16, 1 << 16); 456 mrc_alt_write_mask(DDRPHY, 457 B01PTRCTL1 + 458 bl_grp * DDRIODQ_BL_OFFSET + 459 ch * DDRIODQ_CH_OFFSET, 460 1 << 8, 1 << 8); 461 462 /* RX (PO) */ 463 /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */ 464 mrc_alt_write_mask(DDRPHY, 465 B0VREFCTL + 466 bl_grp * DDRIODQ_BL_OFFSET + 467 ch * DDRIODQ_CH_OFFSET, 468 (0x03 << 2) | (0x0 << 1) | (0x0 << 0), 469 0xff); 470 /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */ 471 mrc_alt_write_mask(DDRPHY, 472 B1VREFCTL + 473 bl_grp * DDRIODQ_BL_OFFSET + 474 ch * DDRIODQ_CH_OFFSET, 475 (0x03 << 2) | (0x0 << 1) | (0x0 << 0), 476 0xff); 477 /* Per-Bit De-Skew Enable */ 478 mrc_alt_write_mask(DDRPHY, 479 B0RXIOBUFCTL + 480 bl_grp * DDRIODQ_BL_OFFSET + 481 ch * DDRIODQ_CH_OFFSET, 482 0, 0x10); 483 /* Per-Bit De-Skew Enable */ 484 mrc_alt_write_mask(DDRPHY, 485 B1RXIOBUFCTL + 486 bl_grp * DDRIODQ_BL_OFFSET + 487 ch * DDRIODQ_CH_OFFSET, 488 0, 0x10); 489 } 490 491 /* CLKEBB */ 492 mrc_alt_write_mask(DDRPHY, 493 CMDOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET, 494 0, 1 << 23); 495 496 /* Enable tristate control of cmd/address bus */ 497 mrc_alt_write_mask(DDRPHY, 498 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET, 499 0, 0x03); 500 501 /* ODT RCOMP */ 502 mrc_alt_write_mask(DDRPHY, 503 CMDRCOMPODT + ch * DDRIOCCC_CH_OFFSET, 504 (0x03 << 5) | (0x03 << 0), 0x3ff); 505 506 /* CMDPM* registers must be programmed in this order */ 507 508 /* Turn On Delays: SFR (regulator), MPLL */ 509 mrc_alt_write_mask(DDRPHY, 510 CMDPMDLYREG4 + ch * DDRIOCCC_CH_OFFSET, 511 0xffffffff, 0xffffffff); 512 /* 513 * Delays: ASSERT_IOBUFACT_to_ALLON0_for_PM_MSG_3, 514 * VREG (MDLL) Turn On, ALLON0_to_DEASSERT_IOBUFACT 515 * for_PM_MSG_gt0, MDLL Turn On 516 */ 517 mrc_alt_write_mask(DDRPHY, 518 CMDPMDLYREG3 + ch * DDRIOCCC_CH_OFFSET, 519 0xfffff616, 0xffffffff); 520 /* MPLL Divider Reset Delays */ 521 mrc_alt_write_mask(DDRPHY, 522 CMDPMDLYREG2 + ch * DDRIOCCC_CH_OFFSET, 523 0xffffffff, 0xffffffff); 524 /* Turn Off Delays: VREG, Staggered MDLL, MDLL, PI */ 525 mrc_alt_write_mask(DDRPHY, 526 CMDPMDLYREG1 + ch * DDRIOCCC_CH_OFFSET, 527 0xffffffff, 0xffffffff); 528 /* Turn On Delays: MPLL, Staggered MDLL, PI, IOBUFACT */ 529 mrc_alt_write_mask(DDRPHY, 530 CMDPMDLYREG0 + ch * DDRIOCCC_CH_OFFSET, 531 0xffffffff, 0xffffffff); 532 /* Allow PUnit signals */ 533 mrc_alt_write_mask(DDRPHY, 534 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET, 535 (0x6 << 8) | (0x1 << 6) | (0x4 << 0), 536 0xffe00f4f); 537 /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */ 538 mrc_alt_write_mask(DDRPHY, 539 CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET, 540 (0x3 << 4) | (0x7 << 0), 0x7f); 541 542 /* CLK-CTL */ 543 mrc_alt_write_mask(DDRPHY, 544 CCOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET, 545 0, 1 << 24); /* CLKEBB */ 546 /* Buffer Enable: CS,CKE,ODT,CLK */ 547 mrc_alt_write_mask(DDRPHY, 548 CCCFGREG0 + ch * DDRIOCCC_CH_OFFSET, 549 0x1f, 0x000ffff1); 550 /* ODT RCOMP */ 551 mrc_alt_write_mask(DDRPHY, 552 CCRCOMPODT + ch * DDRIOCCC_CH_OFFSET, 553 (0x03 << 8) | (0x03 << 0), 0x00001f1f); 554 /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */ 555 mrc_alt_write_mask(DDRPHY, 556 CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET, 557 (0x3 << 4) | (0x7 << 0), 0x7f); 558 559 /* 560 * COMP (RON channel specific) 561 * - DQ/DQS/DM RON: 32 Ohm 562 * - CTRL/CMD RON: 27 Ohm 563 * - CLK RON: 26 Ohm 564 */ 565 /* RCOMP Vref PU/PD */ 566 mrc_alt_write_mask(DDRPHY, 567 DQVREFCH0 + ch * DDRCOMP_CH_OFFSET, 568 (0x08 << 24) | (0x03 << 16), 0x3f3f0000); 569 /* RCOMP Vref PU/PD */ 570 mrc_alt_write_mask(DDRPHY, 571 CMDVREFCH0 + ch * DDRCOMP_CH_OFFSET, 572 (0x0C << 24) | (0x03 << 16), 0x3f3f0000); 573 /* RCOMP Vref PU/PD */ 574 mrc_alt_write_mask(DDRPHY, 575 CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET, 576 (0x0F << 24) | (0x03 << 16), 0x3f3f0000); 577 /* RCOMP Vref PU/PD */ 578 mrc_alt_write_mask(DDRPHY, 579 DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET, 580 (0x08 << 24) | (0x03 << 16), 0x3f3f0000); 581 /* RCOMP Vref PU/PD */ 582 mrc_alt_write_mask(DDRPHY, 583 CTLVREFCH0 + ch * DDRCOMP_CH_OFFSET, 584 (0x0C << 24) | (0x03 << 16), 0x3f3f0000); 585 586 /* DQS Swapped Input Enable */ 587 mrc_alt_write_mask(DDRPHY, 588 COMPEN1CH0 + ch * DDRCOMP_CH_OFFSET, 589 (1 << 19) | (1 << 17), 0xc00ac000); 590 591 /* ODT VREF = 1.5 x 274/360+274 = 0.65V (code of ~50) */ 592 /* ODT Vref PU/PD */ 593 mrc_alt_write_mask(DDRPHY, 594 DQVREFCH0 + ch * DDRCOMP_CH_OFFSET, 595 (0x32 << 8) | (0x03 << 0), 0x00003f3f); 596 /* ODT Vref PU/PD */ 597 mrc_alt_write_mask(DDRPHY, 598 DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET, 599 (0x32 << 8) | (0x03 << 0), 0x00003f3f); 600 /* ODT Vref PU/PD */ 601 mrc_alt_write_mask(DDRPHY, 602 CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET, 603 (0x0E << 8) | (0x05 << 0), 0x00003f3f); 604 605 /* 606 * Slew rate settings are frequency specific, 607 * numbers below are for 800Mhz (speed == 0) 608 * - DQ/DQS/DM/CLK SR: 4V/ns, 609 * - CTRL/CMD SR: 1.5V/ns 610 */ 611 temp = (0x0e << 16) | (0x0e << 12) | (0x08 << 8) | 612 (0x0b << 4) | (0x0b << 0); 613 /* DCOMP Delay Select: CTL,CMD,CLK,DQS,DQ */ 614 mrc_alt_write_mask(DDRPHY, 615 DLYSELCH0 + ch * DDRCOMP_CH_OFFSET, 616 temp, 0x000fffff); 617 /* TCO Vref CLK,DQS,DQ */ 618 mrc_alt_write_mask(DDRPHY, 619 TCOVREFCH0 + ch * DDRCOMP_CH_OFFSET, 620 (0x05 << 16) | (0x05 << 8) | (0x05 << 0), 621 0x003f3f3f); 622 /* ODTCOMP CMD/CTL PU/PD */ 623 mrc_alt_write_mask(DDRPHY, 624 CCBUFODTCH0 + ch * DDRCOMP_CH_OFFSET, 625 (0x03 << 8) | (0x03 << 0), 626 0x00001f1f); 627 /* COMP */ 628 mrc_alt_write_mask(DDRPHY, 629 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET, 630 0, 0xc0000100); 631 632#ifdef BACKUP_COMPS 633 /* DQ COMP Overrides */ 634 /* RCOMP PU */ 635 mrc_alt_write_mask(DDRPHY, 636 DQDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 637 (1 << 31) | (0x0a << 16), 638 0x801f0000); 639 /* RCOMP PD */ 640 mrc_alt_write_mask(DDRPHY, 641 DQDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 642 (1 << 31) | (0x0a << 16), 643 0x801f0000); 644 /* DCOMP PU */ 645 mrc_alt_write_mask(DDRPHY, 646 DQDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 647 (1 << 31) | (0x10 << 16), 648 0x801f0000); 649 /* DCOMP PD */ 650 mrc_alt_write_mask(DDRPHY, 651 DQDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 652 (1 << 31) | (0x10 << 16), 653 0x801f0000); 654 /* ODTCOMP PU */ 655 mrc_alt_write_mask(DDRPHY, 656 DQODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 657 (1 << 31) | (0x0b << 16), 658 0x801f0000); 659 /* ODTCOMP PD */ 660 mrc_alt_write_mask(DDRPHY, 661 DQODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 662 (1 << 31) | (0x0b << 16), 663 0x801f0000); 664 /* TCOCOMP PU */ 665 mrc_alt_write_mask(DDRPHY, 666 DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 667 1 << 31, 1 << 31); 668 /* TCOCOMP PD */ 669 mrc_alt_write_mask(DDRPHY, 670 DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 671 1 << 31, 1 << 31); 672 673 /* DQS COMP Overrides */ 674 /* RCOMP PU */ 675 mrc_alt_write_mask(DDRPHY, 676 DQSDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 677 (1 << 31) | (0x0a << 16), 678 0x801f0000); 679 /* RCOMP PD */ 680 mrc_alt_write_mask(DDRPHY, 681 DQSDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 682 (1 << 31) | (0x0a << 16), 683 0x801f0000); 684 /* DCOMP PU */ 685 mrc_alt_write_mask(DDRPHY, 686 DQSDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 687 (1 << 31) | (0x10 << 16), 688 0x801f0000); 689 /* DCOMP PD */ 690 mrc_alt_write_mask(DDRPHY, 691 DQSDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 692 (1 << 31) | (0x10 << 16), 693 0x801f0000); 694 /* ODTCOMP PU */ 695 mrc_alt_write_mask(DDRPHY, 696 DQSODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 697 (1 << 31) | (0x0b << 16), 698 0x801f0000); 699 /* ODTCOMP PD */ 700 mrc_alt_write_mask(DDRPHY, 701 DQSODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 702 (1 << 31) | (0x0b << 16), 703 0x801f0000); 704 /* TCOCOMP PU */ 705 mrc_alt_write_mask(DDRPHY, 706 DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 707 1 << 31, 1 << 31); 708 /* TCOCOMP PD */ 709 mrc_alt_write_mask(DDRPHY, 710 DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 711 1 << 31, 1 << 31); 712 713 /* CLK COMP Overrides */ 714 /* RCOMP PU */ 715 mrc_alt_write_mask(DDRPHY, 716 CLKDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 717 (1 << 31) | (0x0c << 16), 718 0x801f0000); 719 /* RCOMP PD */ 720 mrc_alt_write_mask(DDRPHY, 721 CLKDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 722 (1 << 31) | (0x0c << 16), 723 0x801f0000); 724 /* DCOMP PU */ 725 mrc_alt_write_mask(DDRPHY, 726 CLKDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 727 (1 << 31) | (0x07 << 16), 728 0x801f0000); 729 /* DCOMP PD */ 730 mrc_alt_write_mask(DDRPHY, 731 CLKDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 732 (1 << 31) | (0x07 << 16), 733 0x801f0000); 734 /* ODTCOMP PU */ 735 mrc_alt_write_mask(DDRPHY, 736 CLKODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 737 (1 << 31) | (0x0b << 16), 738 0x801f0000); 739 /* ODTCOMP PD */ 740 mrc_alt_write_mask(DDRPHY, 741 CLKODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 742 (1 << 31) | (0x0b << 16), 743 0x801f0000); 744 /* TCOCOMP PU */ 745 mrc_alt_write_mask(DDRPHY, 746 CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 747 1 << 31, 1 << 31); 748 /* TCOCOMP PD */ 749 mrc_alt_write_mask(DDRPHY, 750 CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 751 1 << 31, 1 << 31); 752 753 /* CMD COMP Overrides */ 754 /* RCOMP PU */ 755 mrc_alt_write_mask(DDRPHY, 756 CMDDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 757 (1 << 31) | (0x0d << 16), 758 0x803f0000); 759 /* RCOMP PD */ 760 mrc_alt_write_mask(DDRPHY, 761 CMDDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 762 (1 << 31) | (0x0d << 16), 763 0x803f0000); 764 /* DCOMP PU */ 765 mrc_alt_write_mask(DDRPHY, 766 CMDDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 767 (1 << 31) | (0x0a << 16), 768 0x801f0000); 769 /* DCOMP PD */ 770 mrc_alt_write_mask(DDRPHY, 771 CMDDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 772 (1 << 31) | (0x0a << 16), 773 0x801f0000); 774 775 /* CTL COMP Overrides */ 776 /* RCOMP PU */ 777 mrc_alt_write_mask(DDRPHY, 778 CTLDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 779 (1 << 31) | (0x0d << 16), 780 0x803f0000); 781 /* RCOMP PD */ 782 mrc_alt_write_mask(DDRPHY, 783 CTLDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 784 (1 << 31) | (0x0d << 16), 785 0x803f0000); 786 /* DCOMP PU */ 787 mrc_alt_write_mask(DDRPHY, 788 CTLDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 789 (1 << 31) | (0x0a << 16), 790 0x801f0000); 791 /* DCOMP PD */ 792 mrc_alt_write_mask(DDRPHY, 793 CTLDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 794 (1 << 31) | (0x0a << 16), 795 0x801f0000); 796#else 797 /* DQ TCOCOMP Overrides */ 798 /* TCOCOMP PU */ 799 mrc_alt_write_mask(DDRPHY, 800 DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 801 (1 << 31) | (0x1f << 16), 802 0x801f0000); 803 /* TCOCOMP PD */ 804 mrc_alt_write_mask(DDRPHY, 805 DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 806 (1 << 31) | (0x1f << 16), 807 0x801f0000); 808 809 /* DQS TCOCOMP Overrides */ 810 /* TCOCOMP PU */ 811 mrc_alt_write_mask(DDRPHY, 812 DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 813 (1 << 31) | (0x1f << 16), 814 0x801f0000); 815 /* TCOCOMP PD */ 816 mrc_alt_write_mask(DDRPHY, 817 DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 818 (1 << 31) | (0x1f << 16), 819 0x801f0000); 820 821 /* CLK TCOCOMP Overrides */ 822 /* TCOCOMP PU */ 823 mrc_alt_write_mask(DDRPHY, 824 CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 825 (1 << 31) | (0x1f << 16), 826 0x801f0000); 827 /* TCOCOMP PD */ 828 mrc_alt_write_mask(DDRPHY, 829 CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 830 (1 << 31) | (0x1f << 16), 831 0x801f0000); 832#endif 833 834 /* program STATIC delays */ 835#ifdef BACKUP_WCMD 836 set_wcmd(ch, ddr_wcmd[PLATFORM_ID]); 837#else 838 set_wcmd(ch, ddr_wclk[PLATFORM_ID] + HALF_CLK); 839#endif 840 841 for (rk = 0; rk < NUM_RANKS; rk++) { 842 if (mrc_params->rank_enables & (1 << rk)) { 843 set_wclk(ch, rk, ddr_wclk[PLATFORM_ID]); 844#ifdef BACKUP_WCTL 845 set_wctl(ch, rk, ddr_wctl[PLATFORM_ID]); 846#else 847 set_wctl(ch, rk, ddr_wclk[PLATFORM_ID] + HALF_CLK); 848#endif 849 } 850 } 851 } 852 } 853 854 /* COMP (non channel specific) */ 855 /* RCOMP: Dither PU Enable */ 856 mrc_alt_write_mask(DDRPHY, DQANADRVPUCTL, 1 << 30, 1 << 30); 857 /* RCOMP: Dither PD Enable */ 858 mrc_alt_write_mask(DDRPHY, DQANADRVPDCTL, 1 << 30, 1 << 30); 859 /* RCOMP: Dither PU Enable */ 860 mrc_alt_write_mask(DDRPHY, CMDANADRVPUCTL, 1 << 30, 1 << 30); 861 /* RCOMP: Dither PD Enable */ 862 mrc_alt_write_mask(DDRPHY, CMDANADRVPDCTL, 1 << 30, 1 << 30); 863 /* RCOMP: Dither PU Enable */ 864 mrc_alt_write_mask(DDRPHY, CLKANADRVPUCTL, 1 << 30, 1 << 30); 865 /* RCOMP: Dither PD Enable */ 866 mrc_alt_write_mask(DDRPHY, CLKANADRVPDCTL, 1 << 30, 1 << 30); 867 /* RCOMP: Dither PU Enable */ 868 mrc_alt_write_mask(DDRPHY, DQSANADRVPUCTL, 1 << 30, 1 << 30); 869 /* RCOMP: Dither PD Enable */ 870 mrc_alt_write_mask(DDRPHY, DQSANADRVPDCTL, 1 << 30, 1 << 30); 871 /* RCOMP: Dither PU Enable */ 872 mrc_alt_write_mask(DDRPHY, CTLANADRVPUCTL, 1 << 30, 1 << 30); 873 /* RCOMP: Dither PD Enable */ 874 mrc_alt_write_mask(DDRPHY, CTLANADRVPDCTL, 1 << 30, 1 << 30); 875 /* ODT: Dither PU Enable */ 876 mrc_alt_write_mask(DDRPHY, DQANAODTPUCTL, 1 << 30, 1 << 30); 877 /* ODT: Dither PD Enable */ 878 mrc_alt_write_mask(DDRPHY, DQANAODTPDCTL, 1 << 30, 1 << 30); 879 /* ODT: Dither PU Enable */ 880 mrc_alt_write_mask(DDRPHY, CLKANAODTPUCTL, 1 << 30, 1 << 30); 881 /* ODT: Dither PD Enable */ 882 mrc_alt_write_mask(DDRPHY, CLKANAODTPDCTL, 1 << 30, 1 << 30); 883 /* ODT: Dither PU Enable */ 884 mrc_alt_write_mask(DDRPHY, DQSANAODTPUCTL, 1 << 30, 1 << 30); 885 /* ODT: Dither PD Enable */ 886 mrc_alt_write_mask(DDRPHY, DQSANAODTPDCTL, 1 << 30, 1 << 30); 887 /* DCOMP: Dither PU Enable */ 888 mrc_alt_write_mask(DDRPHY, DQANADLYPUCTL, 1 << 30, 1 << 30); 889 /* DCOMP: Dither PD Enable */ 890 mrc_alt_write_mask(DDRPHY, DQANADLYPDCTL, 1 << 30, 1 << 30); 891 /* DCOMP: Dither PU Enable */ 892 mrc_alt_write_mask(DDRPHY, CMDANADLYPUCTL, 1 << 30, 1 << 30); 893 /* DCOMP: Dither PD Enable */ 894 mrc_alt_write_mask(DDRPHY, CMDANADLYPDCTL, 1 << 30, 1 << 30); 895 /* DCOMP: Dither PU Enable */ 896 mrc_alt_write_mask(DDRPHY, CLKANADLYPUCTL, 1 << 30, 1 << 30); 897 /* DCOMP: Dither PD Enable */ 898 mrc_alt_write_mask(DDRPHY, CLKANADLYPDCTL, 1 << 30, 1 << 30); 899 /* DCOMP: Dither PU Enable */ 900 mrc_alt_write_mask(DDRPHY, DQSANADLYPUCTL, 1 << 30, 1 << 30); 901 /* DCOMP: Dither PD Enable */ 902 mrc_alt_write_mask(DDRPHY, DQSANADLYPDCTL, 1 << 30, 1 << 30); 903 /* DCOMP: Dither PU Enable */ 904 mrc_alt_write_mask(DDRPHY, CTLANADLYPUCTL, 1 << 30, 1 << 30); 905 /* DCOMP: Dither PD Enable */ 906 mrc_alt_write_mask(DDRPHY, CTLANADLYPDCTL, 1 << 30, 1 << 30); 907 /* TCO: Dither PU Enable */ 908 mrc_alt_write_mask(DDRPHY, DQANATCOPUCTL, 1 << 30, 1 << 30); 909 /* TCO: Dither PD Enable */ 910 mrc_alt_write_mask(DDRPHY, DQANATCOPDCTL, 1 << 30, 1 << 30); 911 /* TCO: Dither PU Enable */ 912 mrc_alt_write_mask(DDRPHY, CLKANATCOPUCTL, 1 << 30, 1 << 30); 913 /* TCO: Dither PD Enable */ 914 mrc_alt_write_mask(DDRPHY, CLKANATCOPDCTL, 1 << 30, 1 << 30); 915 /* TCO: Dither PU Enable */ 916 mrc_alt_write_mask(DDRPHY, DQSANATCOPUCTL, 1 << 30, 1 << 30); 917 /* TCO: Dither PD Enable */ 918 mrc_alt_write_mask(DDRPHY, DQSANATCOPDCTL, 1 << 30, 1 << 30); 919 /* TCOCOMP: Pulse Count */ 920 mrc_alt_write_mask(DDRPHY, TCOCNTCTRL, 1, 3); 921 /* ODT: CMD/CTL PD/PU */ 922 mrc_alt_write_mask(DDRPHY, CHNLBUFSTATIC, 923 (0x03 << 24) | (0x03 << 16), 0x1f1f0000); 924 /* Set 1us counter */ 925 mrc_alt_write_mask(DDRPHY, MSCNTR, 0x64, 0xff); 926 mrc_alt_write_mask(DDRPHY, LATCH1CTL, 0x1 << 28, 0x70000000); 927 928 /* Release PHY from reset */ 929 mrc_alt_write_mask(DDRPHY, MASTERRSTN, 1, 1); 930 931 /* STEP1 */ 932 mrc_post_code(0x03, 0x11); 933 934 for (ch = 0; ch < NUM_CHANNELS; ch++) { 935 if (mrc_params->channel_enables & (1 << ch)) { 936 /* DQ01-DQ23 */ 937 for (bl_grp = 0; 938 bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2; 939 bl_grp++) { 940 mrc_alt_write_mask(DDRPHY, 941 DQMDLLCTL + 942 bl_grp * DDRIODQ_BL_OFFSET + 943 ch * DDRIODQ_CH_OFFSET, 944 1 << 13, 945 1 << 13); /* Enable VREG */ 946 delay_n(3); 947 } 948 949 /* ECC */ 950 mrc_alt_write_mask(DDRPHY, ECCMDLLCTL, 951 1 << 13, 1 << 13); /* Enable VREG */ 952 delay_n(3); 953 /* CMD */ 954 mrc_alt_write_mask(DDRPHY, 955 CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET, 956 1 << 13, 1 << 13); /* Enable VREG */ 957 delay_n(3); 958 /* CLK-CTL */ 959 mrc_alt_write_mask(DDRPHY, 960 CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET, 961 1 << 13, 1 << 13); /* Enable VREG */ 962 delay_n(3); 963 } 964 } 965 966 /* STEP2 */ 967 mrc_post_code(0x03, 0x12); 968 delay_n(200); 969 970 for (ch = 0; ch < NUM_CHANNELS; ch++) { 971 if (mrc_params->channel_enables & (1 << ch)) { 972 /* DQ01-DQ23 */ 973 for (bl_grp = 0; 974 bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2; 975 bl_grp++) { 976 mrc_alt_write_mask(DDRPHY, 977 DQMDLLCTL + 978 bl_grp * DDRIODQ_BL_OFFSET + 979 ch * DDRIODQ_CH_OFFSET, 980 1 << 17, 981 1 << 17); /* Enable MCDLL */ 982 delay_n(50); 983 } 984 985 /* ECC */ 986 mrc_alt_write_mask(DDRPHY, ECCMDLLCTL, 987 1 << 17, 1 << 17); /* Enable MCDLL */ 988 delay_n(50); 989 /* CMD */ 990 mrc_alt_write_mask(DDRPHY, 991 CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET, 992 1 << 18, 1 << 18); /* Enable MCDLL */ 993 delay_n(50); 994 /* CLK-CTL */ 995 mrc_alt_write_mask(DDRPHY, 996 CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET, 997 1 << 18, 1 << 18); /* Enable MCDLL */ 998 delay_n(50); 999 } 1000 } 1001 1002 /* STEP3: */ 1003 mrc_post_code(0x03, 0x13); 1004 delay_n(100); 1005 1006 for (ch = 0; ch < NUM_CHANNELS; ch++) { 1007 if (mrc_params->channel_enables & (1 << ch)) { 1008 /* DQ01-DQ23 */ 1009 for (bl_grp = 0; 1010 bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2; 1011 bl_grp++) { 1012#ifdef FORCE_16BIT_DDRIO 1013 temp = (bl_grp && 1014 (mrc_params->channel_width == X16)) ? 1015 0x11ff : 0xffff; 1016#else 1017 temp = 0xffff; 1018#endif 1019 /* Enable TXDLL */ 1020 mrc_alt_write_mask(DDRPHY, 1021 DQDLLTXCTL + 1022 bl_grp * DDRIODQ_BL_OFFSET + 1023 ch * DDRIODQ_CH_OFFSET, 1024 temp, 0xffff); 1025 delay_n(3); 1026 /* Enable RXDLL */ 1027 mrc_alt_write_mask(DDRPHY, 1028 DQDLLRXCTL + 1029 bl_grp * DDRIODQ_BL_OFFSET + 1030 ch * DDRIODQ_CH_OFFSET, 1031 0xf, 0xf); 1032 delay_n(3); 1033 /* Enable RXDLL Overrides BL0 */ 1034 mrc_alt_write_mask(DDRPHY, 1035 B0OVRCTL + 1036 bl_grp * DDRIODQ_BL_OFFSET + 1037 ch * DDRIODQ_CH_OFFSET, 1038 0xf, 0xf); 1039 } 1040 1041 /* ECC */ 1042 temp = 0xffff; 1043 mrc_alt_write_mask(DDRPHY, ECCDLLTXCTL, 1044 temp, 0xffff); 1045 delay_n(3); 1046 1047 /* CMD (PO) */ 1048 mrc_alt_write_mask(DDRPHY, 1049 CMDDLLTXCTL + ch * DDRIOCCC_CH_OFFSET, 1050 temp, 0xffff); 1051 delay_n(3); 1052 } 1053 } 1054 1055 /* STEP4 */ 1056 mrc_post_code(0x03, 0x14); 1057 1058 for (ch = 0; ch < NUM_CHANNELS; ch++) { 1059 if (mrc_params->channel_enables & (1 << ch)) { 1060 /* Host To Memory Clock Alignment (HMC) for 800/1066 */ 1061 for (bl_grp = 0; 1062 bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2; 1063 bl_grp++) { 1064 /* CLK_ALIGN_MOD_ID */ 1065 mrc_alt_write_mask(DDRPHY, 1066 DQCLKALIGNREG2 + 1067 bl_grp * DDRIODQ_BL_OFFSET + 1068 ch * DDRIODQ_CH_OFFSET, 1069 bl_grp ? 3 : 1, 1070 0xf); 1071 } 1072 1073 mrc_alt_write_mask(DDRPHY, 1074 ECCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET, 1075 0x2, 0xf); 1076 mrc_alt_write_mask(DDRPHY, 1077 CMDCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET, 1078 0x0, 0xf); 1079 mrc_alt_write_mask(DDRPHY, 1080 CCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET, 1081 0x2, 0xf); 1082 mrc_alt_write_mask(DDRPHY, 1083 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET, 1084 0x20, 0x30); 1085 /* 1086 * NUM_SAMPLES, MAX_SAMPLES, 1087 * MACRO_PI_STEP, MICRO_PI_STEP 1088 */ 1089 mrc_alt_write_mask(DDRPHY, 1090 CMDCLKALIGNREG1 + ch * DDRIOCCC_CH_OFFSET, 1091 (0x18 << 16) | (0x10 << 8) | 1092 (0x8 << 2) | (0x1 << 0), 1093 0x007f7fff); 1094 /* TOTAL_NUM_MODULES, FIRST_U_PARTITION */ 1095 mrc_alt_write_mask(DDRPHY, 1096 CMDCLKALIGNREG2 + ch * DDRIOCCC_CH_OFFSET, 1097 (0x10 << 16) | (0x4 << 8) | (0x2 << 4), 1098 0x001f0ff0); 1099#ifdef HMC_TEST 1100 /* START_CLK_ALIGN=1 */ 1101 mrc_alt_write_mask(DDRPHY, 1102 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET, 1103 1 << 24, 1 << 24); 1104 while (msg_port_alt_read(DDRPHY, 1105 CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET) & 1106 (1 << 24)) 1107 ; /* wait for START_CLK_ALIGN=0 */ 1108#endif 1109 1110 /* Set RD/WR Pointer Seperation & COUNTEN & FIFOPTREN */ 1111 mrc_alt_write_mask(DDRPHY, 1112 CMDPTRREG + ch * DDRIOCCC_CH_OFFSET, 1113 1, 1); /* WRPTRENABLE=1 */ 1114 1115 /* COMP initial */ 1116 /* enable bypass for CLK buffer (PO) */ 1117 mrc_alt_write_mask(DDRPHY, 1118 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET, 1119 1 << 5, 1 << 5); 1120 /* Initial COMP Enable */ 1121 mrc_alt_write_mask(DDRPHY, CMPCTRL, 1, 1); 1122 /* wait for Initial COMP Enable = 0 */ 1123 while (msg_port_alt_read(DDRPHY, CMPCTRL) & 1) 1124 ; 1125 /* disable bypass for CLK buffer (PO) */ 1126 mrc_alt_write_mask(DDRPHY, 1127 COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET, 1128 ~(1 << 5), 1 << 5); 1129 1130 /* IOBUFACT */ 1131 1132 /* STEP4a */ 1133 mrc_alt_write_mask(DDRPHY, 1134 CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET, 1135 1 << 2, 1 << 2); /* IOBUFACTRST_N=1 */ 1136 1137 /* DDRPHY initialization complete */ 1138 mrc_alt_write_mask(DDRPHY, 1139 CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET, 1140 1 << 20, 1 << 20); /* SPID_INIT_COMPLETE=1 */ 1141 } 1142 } 1143 1144 LEAVEFN(); 1145} 1146 1147/* This function performs JEDEC initialization on all enabled channels */ 1148void perform_jedec_init(struct mrc_params *mrc_params) 1149{ 1150 uint8_t twr, wl, rank; 1151 uint32_t tck; 1152 u32 dtr0; 1153 u32 drp; 1154 u32 drmc; 1155 u32 mrs0_cmd = 0; 1156 u32 emrs1_cmd = 0; 1157 u32 emrs2_cmd = 0; 1158 u32 emrs3_cmd = 0; 1159 1160 ENTERFN(); 1161 1162 /* jedec_init starts */ 1163 mrc_post_code(0x04, 0x00); 1164 1165 /* DDR3_RESET_SET=0, DDR3_RESET_RESET=1 */ 1166 mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 2, 0x102); 1167 1168 /* Assert RESET# for 200us */ 1169 delay_u(200); 1170 1171 /* DDR3_RESET_SET=1, DDR3_RESET_RESET=0 */ 1172 mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 0x100, 0x102); 1173 1174 dtr0 = msg_port_read(MEM_CTLR, DTR0); 1175 1176 /* 1177 * Set CKEVAL for populated ranks 1178 * then send NOP to each rank (#4550197) 1179 */ 1180 1181 drp = msg_port_read(MEM_CTLR, DRP); 1182 drp &= 0x3; 1183 1184 drmc = msg_port_read(MEM_CTLR, DRMC); 1185 drmc &= 0xfffffffc; 1186 drmc |= (DRMC_CKEMODE | drp); 1187 1188 msg_port_write(MEM_CTLR, DRMC, drmc); 1189 1190 for (rank = 0; rank < NUM_RANKS; rank++) { 1191 /* Skip to next populated rank */ 1192 if ((mrc_params->rank_enables & (1 << rank)) == 0) 1193 continue; 1194 1195 dram_init_command(DCMD_NOP(rank)); 1196 } 1197 1198 msg_port_write(MEM_CTLR, DRMC, 1199 (mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0)); 1200 1201 /* 1202 * setup for emrs 2 1203 * BIT[15:11] --> Always "0" 1204 * BIT[10:09] --> Rtt_WR: want "Dynamic ODT Off" (0) 1205 * BIT[08] --> Always "0" 1206 * BIT[07] --> SRT: use sr_temp_range 1207 * BIT[06] --> ASR: want "Manual SR Reference" (0) 1208 * BIT[05:03] --> CWL: use oem_tCWL 1209 * BIT[02:00] --> PASR: want "Full Array" (0) 1210 */ 1211 emrs2_cmd |= (2 << 3); 1212 wl = 5 + mrc_params->ddr_speed; 1213 emrs2_cmd |= ((wl - 5) << 9); 1214 emrs2_cmd |= (mrc_params->sr_temp_range << 13); 1215 1216 /* 1217 * setup for emrs 3 1218 * BIT[15:03] --> Always "0" 1219 * BIT[02] --> MPR: want "Normal Operation" (0) 1220 * BIT[01:00] --> MPR_Loc: want "Predefined Pattern" (0) 1221 */ 1222 emrs3_cmd |= (3 << 3); 1223 1224 /* 1225 * setup for emrs 1 1226 * BIT[15:13] --> Always "0" 1227 * BIT[12:12] --> Qoff: want "Output Buffer Enabled" (0) 1228 * BIT[11:11] --> TDQS: want "Disabled" (0) 1229 * BIT[10:10] --> Always "0" 1230 * BIT[09,06,02] --> Rtt_nom: use rtt_nom_value 1231 * BIT[08] --> Always "0" 1232 * BIT[07] --> WR_LVL: want "Disabled" (0) 1233 * BIT[05,01] --> DIC: use ron_value 1234 * BIT[04:03] --> AL: additive latency want "0" (0) 1235 * BIT[00] --> DLL: want "Enable" (0) 1236 * 1237 * (BIT5|BIT1) set Ron value 1238 * 00 --> RZQ/6 (40ohm) 1239 * 01 --> RZQ/7 (34ohm) 1240 * 1* --> RESERVED 1241 * 1242 * (BIT9|BIT6|BIT2) set Rtt_nom value 1243 * 000 --> Disabled 1244 * 001 --> RZQ/4 ( 60ohm) 1245 * 010 --> RZQ/2 (120ohm) 1246 * 011 --> RZQ/6 ( 40ohm) 1247 * 1** --> RESERVED 1248 */ 1249 emrs1_cmd |= (1 << 3); 1250 emrs1_cmd &= ~(1 << 6); 1251 1252 if (mrc_params->ron_value == 0) 1253 emrs1_cmd |= (1 << 7); 1254 else 1255 emrs1_cmd &= ~(1 << 7); 1256 1257 if (mrc_params->rtt_nom_value == 0) 1258 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_40 << 6); 1259 else if (mrc_params->rtt_nom_value == 1) 1260 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_60 << 6); 1261 else if (mrc_params->rtt_nom_value == 2) 1262 emrs1_cmd |= (DDR3_EMRS1_RTTNOM_120 << 6); 1263 1264 /* save MRS1 value (excluding control fields) */ 1265 mrc_params->mrs1 = emrs1_cmd >> 6; 1266 1267 /* 1268 * setup for mrs 0 1269 * BIT[15:13] --> Always "0" 1270 * BIT[12] --> PPD: for Quark (1) 1271 * BIT[11:09] --> WR: use oem_tWR 1272 * BIT[08] --> DLL: want "Reset" (1, self clearing) 1273 * BIT[07] --> MODE: want "Normal" (0) 1274 * BIT[06:04,02] --> CL: use oem_tCAS 1275 * BIT[03] --> RD_BURST_TYPE: want "Interleave" (1) 1276 * BIT[01:00] --> BL: want "8 Fixed" (0) 1277 * WR: 1278 * 0 --> 16 1279 * 1 --> 5 1280 * 2 --> 6 1281 * 3 --> 7 1282 * 4 --> 8 1283 * 5 --> 10 1284 * 6 --> 12 1285 * 7 --> 14 1286 * CL: 1287 * BIT[02:02] "0" if oem_tCAS <= 11 (1866?) 1288 * BIT[06:04] use oem_tCAS-4 1289 */ 1290 mrs0_cmd |= (1 << 14); 1291 mrs0_cmd |= (1 << 18); 1292 mrs0_cmd |= ((((dtr0 >> 12) & 7) + 1) << 10); 1293 1294 tck = t_ck[mrc_params->ddr_speed]; 1295 /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */ 1296 twr = MCEIL(15000, tck); 1297 mrs0_cmd |= ((twr - 4) << 15); 1298 1299 for (rank = 0; rank < NUM_RANKS; rank++) { 1300 /* Skip to next populated rank */ 1301 if ((mrc_params->rank_enables & (1 << rank)) == 0) 1302 continue; 1303 1304 emrs2_cmd |= (rank << 22); 1305 dram_init_command(emrs2_cmd); 1306 1307 emrs3_cmd |= (rank << 22); 1308 dram_init_command(emrs3_cmd); 1309 1310 emrs1_cmd |= (rank << 22); 1311 dram_init_command(emrs1_cmd); 1312 1313 mrs0_cmd |= (rank << 22); 1314 dram_init_command(mrs0_cmd); 1315 1316 dram_init_command(DCMD_ZQCL(rank)); 1317 } 1318 1319 LEAVEFN(); 1320} 1321 1322/* 1323 * Dunit Initialization Complete 1324 * 1325 * Indicates that initialization of the Dunit has completed. 1326 * 1327 * Memory accesses are permitted and maintenance operation begins. 1328 * Until this bit is set to a 1, the memory controller will not accept 1329 * DRAM requests from the MEMORY_MANAGER or HTE. 1330 */ 1331void set_ddr_init_complete(struct mrc_params *mrc_params) 1332{ 1333 u32 dco; 1334 1335 ENTERFN(); 1336 1337 dco = msg_port_read(MEM_CTLR, DCO); 1338 dco &= ~DCO_PMICTL; 1339 dco |= DCO_IC; 1340 msg_port_write(MEM_CTLR, DCO, dco); 1341 1342 LEAVEFN(); 1343} 1344 1345/* 1346 * This function will retrieve relevant timing data 1347 * 1348 * This data will be used on subsequent boots to speed up boot times 1349 * and is required for Suspend To RAM capabilities. 1350 */ 1351void restore_timings(struct mrc_params *mrc_params) 1352{ 1353 uint8_t ch, rk, bl; 1354 const struct mrc_timings *mt = &mrc_params->timings; 1355 1356 for (ch = 0; ch < NUM_CHANNELS; ch++) { 1357 for (rk = 0; rk < NUM_RANKS; rk++) { 1358 for (bl = 0; bl < NUM_BYTE_LANES; bl++) { 1359 set_rcvn(ch, rk, bl, mt->rcvn[ch][rk][bl]); 1360 set_rdqs(ch, rk, bl, mt->rdqs[ch][rk][bl]); 1361 set_wdqs(ch, rk, bl, mt->wdqs[ch][rk][bl]); 1362 set_wdq(ch, rk, bl, mt->wdq[ch][rk][bl]); 1363 if (rk == 0) { 1364 /* VREF (RANK0 only) */ 1365 set_vref(ch, bl, mt->vref[ch][bl]); 1366 } 1367 } 1368 set_wctl(ch, rk, mt->wctl[ch][rk]); 1369 } 1370 set_wcmd(ch, mt->wcmd[ch]); 1371 } 1372} 1373 1374/* 1375 * Configure default settings normally set as part of read training 1376 * 1377 * Some defaults have to be set earlier as they may affect earlier 1378 * training steps. 1379 */ 1380void default_timings(struct mrc_params *mrc_params) 1381{ 1382 uint8_t ch, rk, bl; 1383 1384 for (ch = 0; ch < NUM_CHANNELS; ch++) { 1385 for (rk = 0; rk < NUM_RANKS; rk++) { 1386 for (bl = 0; bl < NUM_BYTE_LANES; bl++) { 1387 set_rdqs(ch, rk, bl, 24); 1388 if (rk == 0) { 1389 /* VREF (RANK0 only) */ 1390 set_vref(ch, bl, 32); 1391 } 1392 } 1393 } 1394 } 1395} 1396 1397/* 1398 * This function will perform our RCVEN Calibration Algorithm. 1399 * We will only use the 2xCLK domain timings to perform RCVEN Calibration. 1400 * All byte lanes will be calibrated "simultaneously" per channel per rank. 1401 */ 1402void rcvn_cal(struct mrc_params *mrc_params) 1403{ 1404 uint8_t ch; /* channel counter */ 1405 uint8_t rk; /* rank counter */ 1406 uint8_t bl; /* byte lane counter */ 1407 uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1; 1408 1409#ifdef R2R_SHARING 1410 /* used to find placement for rank2rank sharing configs */ 1411 uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES]; 1412#ifndef BACKUP_RCVN 1413 /* used to find placement for rank2rank sharing configs */ 1414 uint32_t num_ranks_enabled = 0; 1415#endif 1416#endif 1417 1418#ifdef BACKUP_RCVN 1419#else 1420 uint32_t temp; 1421 /* absolute PI value to be programmed on the byte lane */ 1422 uint32_t delay[NUM_BYTE_LANES]; 1423 u32 dtr1, dtr1_save; 1424#endif 1425 1426 ENTERFN(); 1427 1428 /* rcvn_cal starts */ 1429 mrc_post_code(0x05, 0x00); 1430 1431#ifndef BACKUP_RCVN 1432 /* need separate burst to sample DQS preamble */ 1433 dtr1 = msg_port_read(MEM_CTLR, DTR1); 1434 dtr1_save = dtr1; 1435 dtr1 |= DTR1_TCCD_12CLK; 1436 msg_port_write(MEM_CTLR, DTR1, dtr1); 1437#endif 1438 1439#ifdef R2R_SHARING 1440 /* need to set "final_delay[][]" elements to "0" */ 1441 memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay)); 1442#endif 1443 1444 /* loop through each enabled channel */ 1445 for (ch = 0; ch < NUM_CHANNELS; ch++) { 1446 if (mrc_params->channel_enables & (1 << ch)) { 1447 /* perform RCVEN Calibration on a per rank basis */ 1448 for (rk = 0; rk < NUM_RANKS; rk++) { 1449 if (mrc_params->rank_enables & (1 << rk)) { 1450 /* 1451 * POST_CODE here indicates the current 1452 * channel and rank being calibrated 1453 */ 1454 mrc_post_code(0x05, 0x10 + ((ch << 4) | rk)); 1455 1456#ifdef BACKUP_RCVN 1457 /* et hard-coded timing values */ 1458 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) 1459 set_rcvn(ch, rk, bl, ddr_rcvn[PLATFORM_ID]); 1460#else 1461 /* enable FIFORST */ 1462 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) { 1463 mrc_alt_write_mask(DDRPHY, 1464 B01PTRCTL1 + 1465 (bl >> 1) * DDRIODQ_BL_OFFSET + 1466 ch * DDRIODQ_CH_OFFSET, 1467 0, 1 << 8); 1468 } 1469 /* initialize the starting delay to 128 PI (cas +1 CLK) */ 1470 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { 1471 /* 1x CLK domain timing is cas-4 */ 1472 delay[bl] = (4 + 1) * FULL_CLK; 1473 1474 set_rcvn(ch, rk, bl, delay[bl]); 1475 } 1476 1477 /* now find the rising edge */ 1478 find_rising_edge(mrc_params, delay, ch, rk, true); 1479 1480 /* Now increase delay by 32 PI (1/4 CLK) to place in center of high pulse */ 1481 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { 1482 delay[bl] += QRTR_CLK; 1483 set_rcvn(ch, rk, bl, delay[bl]); 1484 } 1485 /* Now decrement delay by 128 PI (1 CLK) until we sample a "0" */ 1486 do { 1487 temp = sample_dqs(mrc_params, ch, rk, true); 1488 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { 1489 if (temp & (1 << bl)) { 1490 if (delay[bl] >= FULL_CLK) { 1491 delay[bl] -= FULL_CLK; 1492 set_rcvn(ch, rk, bl, delay[bl]); 1493 } else { 1494 /* not enough delay */ 1495 training_message(ch, rk, bl); 1496 mrc_post_code(0xee, 0x50); 1497 } 1498 } 1499 } 1500 } while (temp & 0xff); 1501 1502#ifdef R2R_SHARING 1503 /* increment "num_ranks_enabled" */ 1504 num_ranks_enabled++; 1505 /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */ 1506 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { 1507 delay[bl] += QRTR_CLK; 1508 /* add "delay[]" values to "final_delay[][]" for rolling average */ 1509 final_delay[ch][bl] += delay[bl]; 1510 /* set timing based on rolling average values */ 1511 set_rcvn(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled); 1512 } 1513#else 1514 /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */ 1515 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { 1516 delay[bl] += QRTR_CLK; 1517 set_rcvn(ch, rk, bl, delay[bl]); 1518 } 1519#endif 1520 1521 /* disable FIFORST */ 1522 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) { 1523 mrc_alt_write_mask(DDRPHY, 1524 B01PTRCTL1 + 1525 (bl >> 1) * DDRIODQ_BL_OFFSET + 1526 ch * DDRIODQ_CH_OFFSET, 1527 1 << 8, 1 << 8); 1528 } 1529#endif 1530 } 1531 } 1532 } 1533 } 1534 1535#ifndef BACKUP_RCVN 1536 /* restore original */ 1537 msg_port_write(MEM_CTLR, DTR1, dtr1_save); 1538#endif 1539 1540 LEAVEFN(); 1541} 1542 1543/* 1544 * This function will perform the Write Levelling algorithm 1545 * (align WCLK and WDQS). 1546 * 1547 * This algorithm will act on each rank in each channel separately. 1548 */ 1549void wr_level(struct mrc_params *mrc_params) 1550{ 1551 uint8_t ch; /* channel counter */ 1552 uint8_t rk; /* rank counter */ 1553 uint8_t bl; /* byte lane counter */ 1554 uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1; 1555 1556#ifdef R2R_SHARING 1557 /* used to find placement for rank2rank sharing configs */ 1558 uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES]; 1559#ifndef BACKUP_WDQS 1560 /* used to find placement for rank2rank sharing configs */ 1561 uint32_t num_ranks_enabled = 0; 1562#endif 1563#endif 1564 1565#ifdef BACKUP_WDQS 1566#else 1567 /* determines stop condition for CRS_WR_LVL */ 1568 bool all_edges_found; 1569 /* absolute PI value to be programmed on the byte lane */ 1570 uint32_t delay[NUM_BYTE_LANES]; 1571 /* 1572 * static makes it so the data is loaded in the heap once by shadow(), 1573 * where non-static copies the data onto the stack every time this 1574 * function is called 1575 */ 1576 uint32_t address; /* address to be checked during COARSE_WR_LVL */ 1577 u32 dtr4, dtr4_save; 1578#endif 1579 1580 ENTERFN(); 1581 1582 /* wr_level starts */ 1583 mrc_post_code(0x06, 0x00); 1584 1585#ifdef R2R_SHARING 1586 /* need to set "final_delay[][]" elements to "0" */ 1587 memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay)); 1588#endif 1589 1590 /* loop through each enabled channel */ 1591 for (ch = 0; ch < NUM_CHANNELS; ch++) { 1592 if (mrc_params->channel_enables & (1 << ch)) { 1593 /* perform WRITE LEVELING algorithm on a per rank basis */ 1594 for (rk = 0; rk < NUM_RANKS; rk++) { 1595 if (mrc_params->rank_enables & (1 << rk)) { 1596 /* 1597 * POST_CODE here indicates the current 1598 * rank and channel being calibrated 1599 */ 1600 mrc_post_code(0x06, 0x10 + ((ch << 4) | rk)); 1601 1602#ifdef BACKUP_WDQS 1603 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { 1604 set_wdqs(ch, rk, bl, ddr_wdqs[PLATFORM_ID]); 1605 set_wdq(ch, rk, bl, ddr_wdqs[PLATFORM_ID] - QRTR_CLK); 1606 } 1607#else 1608 /* 1609 * perform a single PRECHARGE_ALL command to 1610 * make DRAM state machine go to IDLE state 1611 */ 1612 dram_init_command(DCMD_PREA(rk)); 1613 1614 /* 1615 * enable Write Levelling Mode 1616 * (EMRS1 w/ Write Levelling Mode Enable) 1617 */ 1618 dram_init_command(DCMD_MRS1(rk, 0x82)); 1619 1620 /* 1621 * set ODT DRAM Full Time Termination 1622 * disable in MCU 1623 */ 1624 1625 dtr4 = msg_port_read(MEM_CTLR, DTR4); 1626 dtr4_save = dtr4; 1627 dtr4 |= DTR4_ODTDIS; 1628 msg_port_write(MEM_CTLR, DTR4, dtr4); 1629 1630 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) { 1631 /* 1632 * Enable Sandy Bridge Mode (WDQ Tri-State) & 1633 * Ensure 5 WDQS pulses during Write Leveling 1634 */ 1635 mrc_alt_write_mask(DDRPHY, 1636 DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch, 1637 0x10000154, 1638 0x100003fc); 1639 } 1640 1641 /* Write Leveling Mode enabled in IO */ 1642 mrc_alt_write_mask(DDRPHY, 1643 CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch, 1644 1 << 16, 1 << 16); 1645 1646 /* Initialize the starting delay to WCLK */ 1647 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { 1648 /* 1649 * CLK0 --> RK0 1650 * CLK1 --> RK1 1651 */ 1652 delay[bl] = get_wclk(ch, rk); 1653 1654 set_wdqs(ch, rk, bl, delay[bl]); 1655 } 1656 1657 /* now find the rising edge */ 1658 find_rising_edge(mrc_params, delay, ch, rk, false); 1659 1660 /* disable Write Levelling Mode */ 1661 mrc_alt_write_mask(DDRPHY, 1662 CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch, 1663 0, 1 << 16); 1664 1665 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) { 1666 /* Disable Sandy Bridge Mode & Ensure 4 WDQS pulses during normal operation */ 1667 mrc_alt_write_mask(DDRPHY, 1668 DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch, 1669 0x00000154, 1670 0x100003fc); 1671 } 1672 1673 /* restore original DTR4 */ 1674 msg_port_write(MEM_CTLR, DTR4, dtr4_save); 1675 1676 /* 1677 * restore original value 1678 * (Write Levelling Mode Disable) 1679 */ 1680 dram_init_command(DCMD_MRS1(rk, mrc_params->mrs1)); 1681 1682 /* 1683 * perform a single PRECHARGE_ALL command to 1684 * make DRAM state machine go to IDLE state 1685 */ 1686 dram_init_command(DCMD_PREA(rk)); 1687 1688 mrc_post_code(0x06, 0x30 + ((ch << 4) | rk)); 1689 1690 /* 1691 * COARSE WRITE LEVEL: 1692 * check that we're on the correct clock edge 1693 */ 1694 1695 /* hte reconfiguration request */ 1696 mrc_params->hte_setup = 1; 1697 1698 /* start CRS_WR_LVL with WDQS = WDQS + 128 PI */ 1699 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { 1700 delay[bl] = get_wdqs(ch, rk, bl) + FULL_CLK; 1701 set_wdqs(ch, rk, bl, delay[bl]); 1702 /* 1703 * program WDQ timings based on WDQS 1704 * (WDQ = WDQS - 32 PI) 1705 */ 1706 set_wdq(ch, rk, bl, (delay[bl] - QRTR_CLK)); 1707 } 1708 1709 /* get an address in the targeted channel/rank */ 1710 address = get_addr(ch, rk); 1711 do { 1712 uint32_t coarse_result = 0x00; 1713 uint32_t coarse_result_mask = byte_lane_mask(mrc_params); 1714 /* assume pass */ 1715 all_edges_found = true; 1716 1717 mrc_params->hte_setup = 1; 1718 coarse_result = check_rw_coarse(mrc_params, address); 1719 1720 /* check for failures and margin the byte lane back 128 PI (1 CLK) */ 1721 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { 1722 if (coarse_result & (coarse_result_mask << bl)) { 1723 all_edges_found = false; 1724 delay[bl] -= FULL_CLK; 1725 set_wdqs(ch, rk, bl, delay[bl]); 1726 /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */ 1727 set_wdq(ch, rk, bl, delay[bl] - QRTR_CLK); 1728 } 1729 } 1730 } while (!all_edges_found); 1731 1732#ifdef R2R_SHARING 1733 /* increment "num_ranks_enabled" */ 1734 num_ranks_enabled++; 1735 /* accumulate "final_delay[][]" values from "delay[]" values for rolling average */ 1736 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { 1737 final_delay[ch][bl] += delay[bl]; 1738 set_wdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled); 1739 /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */ 1740 set_wdq(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled - QRTR_CLK); 1741 } 1742#endif 1743#endif 1744 } 1745 } 1746 } 1747 } 1748 1749 LEAVEFN(); 1750} 1751 1752void prog_page_ctrl(struct mrc_params *mrc_params) 1753{ 1754 u32 dpmc0; 1755 1756 ENTERFN(); 1757 1758 dpmc0 = msg_port_read(MEM_CTLR, DPMC0); 1759 dpmc0 &= ~DPMC0_PCLSTO_MASK; 1760 dpmc0 |= (4 << 16); 1761 dpmc0 |= DPMC0_PREAPWDEN; 1762 msg_port_write(MEM_CTLR, DPMC0, dpmc0); 1763} 1764 1765/* 1766 * This function will perform the READ TRAINING Algorithm on all 1767 * channels/ranks/byte_lanes simultaneously to minimize execution time. 1768 * 1769 * The idea here is to train the VREF and RDQS (and eventually RDQ) values 1770 * to achieve maximum READ margins. The algorithm will first determine the 1771 * X coordinate (RDQS setting). This is done by collapsing the VREF eye 1772 * until we find a minimum required RDQS eye for VREF_MIN and VREF_MAX. 1773 * Then we take the averages of the RDQS eye at VREF_MIN and VREF_MAX, 1774 * then average those; this will be the final X coordinate. The algorithm 1775 * will then determine the Y coordinate (VREF setting). This is done by 1776 * collapsing the RDQS eye until we find a minimum required VREF eye for 1777 * RDQS_MIN and RDQS_MAX. Then we take the averages of the VREF eye at 1778 * RDQS_MIN and RDQS_MAX, then average those; this will be the final Y 1779 * coordinate. 1780 * 1781 * NOTE: this algorithm assumes the eye curves have a one-to-one relationship, 1782 * meaning for each X the curve has only one Y and vice-a-versa. 1783 */ 1784void rd_train(struct mrc_params *mrc_params) 1785{ 1786 uint8_t ch; /* channel counter */ 1787 uint8_t rk; /* rank counter */ 1788 uint8_t bl; /* byte lane counter */ 1789 uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1; 1790#ifdef BACKUP_RDQS 1791#else 1792 uint8_t side_x; /* tracks LEFT/RIGHT approach vectors */ 1793 uint8_t side_y; /* tracks BOTTOM/TOP approach vectors */ 1794 /* X coordinate data (passing RDQS values) for approach vectors */ 1795 uint8_t x_coordinate[2][2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES]; 1796 /* Y coordinate data (passing VREF values) for approach vectors */ 1797 uint8_t y_coordinate[2][2][NUM_CHANNELS][NUM_BYTE_LANES]; 1798 /* centered X (RDQS) */ 1799 uint8_t x_center[NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES]; 1800 /* centered Y (VREF) */ 1801 uint8_t y_center[NUM_CHANNELS][NUM_BYTE_LANES]; 1802 uint32_t address; /* target address for check_bls_ex() */ 1803 uint32_t result; /* result of check_bls_ex() */ 1804 uint32_t bl_mask; /* byte lane mask for result checking */ 1805#ifdef R2R_SHARING 1806 /* used to find placement for rank2rank sharing configs */ 1807 uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES]; 1808 /* used to find placement for rank2rank sharing configs */ 1809 uint32_t num_ranks_enabled = 0; 1810#endif 1811#endif 1812 1813 /* rd_train starts */ 1814 mrc_post_code(0x07, 0x00); 1815 1816 ENTERFN(); 1817 1818#ifdef BACKUP_RDQS 1819 for (ch = 0; ch < NUM_CHANNELS; ch++) { 1820 if (mrc_params->channel_enables & (1 << ch)) { 1821 for (rk = 0; rk < NUM_RANKS; rk++) { 1822 if (mrc_params->rank_enables & (1 << rk)) { 1823 for (bl = 0; 1824 bl < NUM_BYTE_LANES / bl_divisor; 1825 bl++) { 1826 set_rdqs(ch, rk, bl, ddr_rdqs[PLATFORM_ID]); 1827 } 1828 } 1829 } 1830 } 1831 } 1832#else 1833 /* initialize x/y_coordinate arrays */ 1834 for (ch = 0; ch < NUM_CHANNELS; ch++) { 1835 if (mrc_params->channel_enables & (1 << ch)) { 1836 for (rk = 0; rk < NUM_RANKS; rk++) { 1837 if (mrc_params->rank_enables & (1 << rk)) { 1838 for (bl = 0; 1839 bl < NUM_BYTE_LANES / bl_divisor; 1840 bl++) { 1841 /* x_coordinate */ 1842 x_coordinate[L][B][ch][rk][bl] = RDQS_MIN; 1843 x_coordinate[R][B][ch][rk][bl] = RDQS_MAX; 1844 x_coordinate[L][T][ch][rk][bl] = RDQS_MIN; 1845 x_coordinate[R][T][ch][rk][bl] = RDQS_MAX; 1846 /* y_coordinate */ 1847 y_coordinate[L][B][ch][bl] = VREF_MIN; 1848 y_coordinate[R][B][ch][bl] = VREF_MIN; 1849 y_coordinate[L][T][ch][bl] = VREF_MAX; 1850 y_coordinate[R][T][ch][bl] = VREF_MAX; 1851 } 1852 } 1853 } 1854 } 1855 } 1856 1857 /* initialize other variables */ 1858 bl_mask = byte_lane_mask(mrc_params); 1859 address = get_addr(0, 0); 1860 1861#ifdef R2R_SHARING 1862 /* need to set "final_delay[][]" elements to "0" */ 1863 memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay)); 1864#endif 1865 1866 /* look for passing coordinates */ 1867 for (side_y = B; side_y <= T; side_y++) { 1868 for (side_x = L; side_x <= R; side_x++) { 1869 mrc_post_code(0x07, 0x10 + side_y * 2 + side_x); 1870 1871 /* find passing values */ 1872 for (ch = 0; ch < NUM_CHANNELS; ch++) { 1873 if (mrc_params->channel_enables & (0x1 << ch)) { 1874 for (rk = 0; rk < NUM_RANKS; rk++) { 1875 if (mrc_params->rank_enables & 1876 (0x1 << rk)) { 1877 /* set x/y_coordinate search starting settings */ 1878 for (bl = 0; 1879 bl < NUM_BYTE_LANES / bl_divisor; 1880 bl++) { 1881 set_rdqs(ch, rk, bl, 1882 x_coordinate[side_x][side_y][ch][rk][bl]); 1883 set_vref(ch, bl, 1884 y_coordinate[side_x][side_y][ch][bl]); 1885 } 1886 1887 /* get an address in the target channel/rank */ 1888 address = get_addr(ch, rk); 1889 1890 /* request HTE reconfiguration */ 1891 mrc_params->hte_setup = 1; 1892 1893 /* test the settings */ 1894 do { 1895 /* result[07:00] == failing byte lane (MAX 8) */ 1896 result = check_bls_ex(mrc_params, address); 1897 1898 /* check for failures */ 1899 if (result & 0xff) { 1900 /* at least 1 byte lane failed */ 1901 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { 1902 if (result & 1903 (bl_mask << bl)) { 1904 /* adjust the RDQS values accordingly */ 1905 if (side_x == L) 1906 x_coordinate[L][side_y][ch][rk][bl] += RDQS_STEP; 1907 else 1908 x_coordinate[R][side_y][ch][rk][bl] -= RDQS_STEP; 1909 1910 /* check that we haven't closed the RDQS_EYE too much */ 1911 if ((x_coordinate[L][side_y][ch][rk][bl] > (RDQS_MAX - MIN_RDQS_EYE)) || 1912 (x_coordinate[R][side_y][ch][rk][bl] < (RDQS_MIN + MIN_RDQS_EYE)) || 1913 (x_coordinate[L][side_y][ch][rk][bl] == 1914 x_coordinate[R][side_y][ch][rk][bl])) { 1915 /* 1916 * not enough RDQS margin available at this VREF 1917 * update VREF values accordingly 1918 */ 1919 if (side_y == B) 1920 y_coordinate[side_x][B][ch][bl] += VREF_STEP; 1921 else 1922 y_coordinate[side_x][T][ch][bl] -= VREF_STEP; 1923 1924 /* check that we haven't closed the VREF_EYE too much */ 1925 if ((y_coordinate[side_x][B][ch][bl] > (VREF_MAX - MIN_VREF_EYE)) || 1926 (y_coordinate[side_x][T][ch][bl] < (VREF_MIN + MIN_VREF_EYE)) || 1927 (y_coordinate[side_x][B][ch][bl] == y_coordinate[side_x][T][ch][bl])) { 1928 /* VREF_EYE collapsed below MIN_VREF_EYE */ 1929 training_message(ch, rk, bl); 1930 mrc_post_code(0xEE, 0x70 + side_y * 2 + side_x); 1931 } else { 1932 /* update the VREF setting */ 1933 set_vref(ch, bl, y_coordinate[side_x][side_y][ch][bl]); 1934 /* reset the X coordinate to begin the search at the new VREF */ 1935 x_coordinate[side_x][side_y][ch][rk][bl] = 1936 (side_x == L) ? RDQS_MIN : RDQS_MAX; 1937 } 1938 } 1939 1940 /* update the RDQS setting */ 1941 set_rdqs(ch, rk, bl, x_coordinate[side_x][side_y][ch][rk][bl]); 1942 } 1943 } 1944 } 1945 } while (result & 0xff); 1946 } 1947 } 1948 } 1949 } 1950 } 1951 } 1952 1953 mrc_post_code(0x07, 0x20); 1954 1955 /* find final RDQS (X coordinate) & final VREF (Y coordinate) */ 1956 for (ch = 0; ch < NUM_CHANNELS; ch++) { 1957 if (mrc_params->channel_enables & (1 << ch)) { 1958 for (rk = 0; rk < NUM_RANKS; rk++) { 1959 if (mrc_params->rank_enables & (1 << rk)) { 1960 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { 1961 uint32_t temp1; 1962 uint32_t temp2; 1963 1964 /* x_coordinate */ 1965 DPF(D_INFO, 1966 "RDQS T/B eye rank%d lane%d : %d-%d %d-%d\n", 1967 rk, bl, 1968 x_coordinate[L][T][ch][rk][bl], 1969 x_coordinate[R][T][ch][rk][bl], 1970 x_coordinate[L][B][ch][rk][bl], 1971 x_coordinate[R][B][ch][rk][bl]); 1972 1973 /* average the TOP side LEFT & RIGHT values */ 1974 temp1 = (x_coordinate[R][T][ch][rk][bl] + x_coordinate[L][T][ch][rk][bl]) / 2; 1975 /* average the BOTTOM side LEFT & RIGHT values */ 1976 temp2 = (x_coordinate[R][B][ch][rk][bl] + x_coordinate[L][B][ch][rk][bl]) / 2; 1977 /* average the above averages */ 1978 x_center[ch][rk][bl] = (uint8_t) ((temp1 + temp2) / 2); 1979 1980 /* y_coordinate */ 1981 DPF(D_INFO, 1982 "VREF R/L eye lane%d : %d-%d %d-%d\n", 1983 bl, 1984 y_coordinate[R][B][ch][bl], 1985 y_coordinate[R][T][ch][bl], 1986 y_coordinate[L][B][ch][bl], 1987 y_coordinate[L][T][ch][bl]); 1988 1989 /* average the RIGHT side TOP & BOTTOM values */ 1990 temp1 = (y_coordinate[R][T][ch][bl] + y_coordinate[R][B][ch][bl]) / 2; 1991 /* average the LEFT side TOP & BOTTOM values */ 1992 temp2 = (y_coordinate[L][T][ch][bl] + y_coordinate[L][B][ch][bl]) / 2; 1993 /* average the above averages */ 1994 y_center[ch][bl] = (uint8_t) ((temp1 + temp2) / 2); 1995 } 1996 } 1997 } 1998 } 1999 } 2000 2001#ifdef RX_EYE_CHECK 2002 /* perform an eye check */ 2003 for (side_y = B; side_y <= T; side_y++) { 2004 for (side_x = L; side_x <= R; side_x++) { 2005 mrc_post_code(0x07, 0x30 + side_y * 2 + side_x); 2006 2007 /* update the settings for the eye check */ 2008 for (ch = 0; ch < NUM_CHANNELS; ch++) { 2009 if (mrc_params->channel_enables & (1 << ch)) { 2010 for (rk = 0; rk < NUM_RANKS; rk++) { 2011 if (mrc_params->rank_enables & (1 << rk)) { 2012 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { 2013 if (side_x == L) 2014 set_rdqs(ch, rk, bl, x_center[ch][rk][bl] - (MIN_RDQS_EYE / 2)); 2015 else 2016 set_rdqs(ch, rk, bl, x_center[ch][rk][bl] + (MIN_RDQS_EYE / 2)); 2017 2018 if (side_y == B) 2019 set_vref(ch, bl, y_center[ch][bl] - (MIN_VREF_EYE / 2)); 2020 else 2021 set_vref(ch, bl, y_center[ch][bl] + (MIN_VREF_EYE / 2)); 2022 } 2023 } 2024 } 2025 } 2026 } 2027 2028 /* request HTE reconfiguration */ 2029 mrc_params->hte_setup = 1; 2030 2031 /* check the eye */ 2032 if (check_bls_ex(mrc_params, address) & 0xff) { 2033 /* one or more byte lanes failed */ 2034 mrc_post_code(0xee, 0x74 + side_x * 2 + side_y); 2035 } 2036 } 2037 } 2038#endif 2039 2040 mrc_post_code(0x07, 0x40); 2041 2042 /* set final placements */ 2043 for (ch = 0; ch < NUM_CHANNELS; ch++) { 2044 if (mrc_params->channel_enables & (1 << ch)) { 2045 for (rk = 0; rk < NUM_RANKS; rk++) { 2046 if (mrc_params->rank_enables & (1 << rk)) { 2047#ifdef R2R_SHARING 2048 /* increment "num_ranks_enabled" */ 2049 num_ranks_enabled++; 2050#endif 2051 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { 2052 /* x_coordinate */ 2053#ifdef R2R_SHARING 2054 final_delay[ch][bl] += x_center[ch][rk][bl]; 2055 set_rdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled); 2056#else 2057 set_rdqs(ch, rk, bl, x_center[ch][rk][bl]); 2058#endif 2059 /* y_coordinate */ 2060 set_vref(ch, bl, y_center[ch][bl]); 2061 } 2062 } 2063 } 2064 } 2065 } 2066#endif 2067 2068 LEAVEFN(); 2069} 2070 2071/* 2072 * This function will perform the WRITE TRAINING Algorithm on all 2073 * channels/ranks/byte_lanes simultaneously to minimize execution time. 2074 * 2075 * The idea here is to train the WDQ timings to achieve maximum WRITE margins. 2076 * The algorithm will start with WDQ at the current WDQ setting (tracks WDQS 2077 * in WR_LVL) +/- 32 PIs (+/- 1/4 CLK) and collapse the eye until all data 2078 * patterns pass. This is because WDQS will be aligned to WCLK by the 2079 * Write Leveling algorithm and WDQ will only ever have a 1/2 CLK window 2080 * of validity. 2081 */ 2082void wr_train(struct mrc_params *mrc_params) 2083{ 2084 uint8_t ch; /* channel counter */ 2085 uint8_t rk; /* rank counter */ 2086 uint8_t bl; /* byte lane counter */ 2087 uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1; 2088#ifdef BACKUP_WDQ 2089#else 2090 uint8_t side; /* LEFT/RIGHT side indicator (0=L, 1=R) */ 2091 uint32_t temp; /* temporary DWORD */ 2092 /* 2 arrays, for L & R side passing delays */ 2093 uint32_t delay[2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES]; 2094 uint32_t address; /* target address for check_bls_ex() */ 2095 uint32_t result; /* result of check_bls_ex() */ 2096 uint32_t bl_mask; /* byte lane mask for result checking */ 2097#ifdef R2R_SHARING 2098 /* used to find placement for rank2rank sharing configs */ 2099 uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES]; 2100 /* used to find placement for rank2rank sharing configs */ 2101 uint32_t num_ranks_enabled = 0; 2102#endif 2103#endif 2104 2105 /* wr_train starts */ 2106 mrc_post_code(0x08, 0x00); 2107 2108 ENTERFN(); 2109 2110#ifdef BACKUP_WDQ 2111 for (ch = 0; ch < NUM_CHANNELS; ch++) { 2112 if (mrc_params->channel_enables & (1 << ch)) { 2113 for (rk = 0; rk < NUM_RANKS; rk++) { 2114 if (mrc_params->rank_enables & (1 << rk)) { 2115 for (bl = 0; 2116 bl < NUM_BYTE_LANES / bl_divisor; 2117 bl++) { 2118 set_wdq(ch, rk, bl, ddr_wdq[PLATFORM_ID]); 2119 } 2120 } 2121 } 2122 } 2123 } 2124#else 2125 /* initialize "delay" */ 2126 for (ch = 0; ch < NUM_CHANNELS; ch++) { 2127 if (mrc_params->channel_enables & (1 << ch)) { 2128 for (rk = 0; rk < NUM_RANKS; rk++) { 2129 if (mrc_params->rank_enables & (1 << rk)) { 2130 for (bl = 0; 2131 bl < NUM_BYTE_LANES / bl_divisor; 2132 bl++) { 2133 /* 2134 * want to start with 2135 * WDQ = (WDQS - QRTR_CLK) 2136 * +/- QRTR_CLK 2137 */ 2138 temp = get_wdqs(ch, rk, bl) - QRTR_CLK; 2139 delay[L][ch][rk][bl] = temp - QRTR_CLK; 2140 delay[R][ch][rk][bl] = temp + QRTR_CLK; 2141 } 2142 } 2143 } 2144 } 2145 } 2146 2147 /* initialize other variables */ 2148 bl_mask = byte_lane_mask(mrc_params); 2149 address = get_addr(0, 0); 2150 2151#ifdef R2R_SHARING 2152 /* need to set "final_delay[][]" elements to "0" */ 2153 memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay)); 2154#endif 2155 2156 /* 2157 * start algorithm on the LEFT side and train each channel/bl 2158 * until no failures are observed, then repeat for the RIGHT side. 2159 */ 2160 for (side = L; side <= R; side++) { 2161 mrc_post_code(0x08, 0x10 + side); 2162 2163 /* set starting values */ 2164 for (ch = 0; ch < NUM_CHANNELS; ch++) { 2165 if (mrc_params->channel_enables & (1 << ch)) { 2166 for (rk = 0; rk < NUM_RANKS; rk++) { 2167 if (mrc_params->rank_enables & 2168 (1 << rk)) { 2169 for (bl = 0; 2170 bl < NUM_BYTE_LANES / bl_divisor; 2171 bl++) { 2172 set_wdq(ch, rk, bl, delay[side][ch][rk][bl]); 2173 } 2174 } 2175 } 2176 } 2177 } 2178 2179 /* find passing values */ 2180 for (ch = 0; ch < NUM_CHANNELS; ch++) { 2181 if (mrc_params->channel_enables & (1 << ch)) { 2182 for (rk = 0; rk < NUM_RANKS; rk++) { 2183 if (mrc_params->rank_enables & 2184 (1 << rk)) { 2185 /* get an address in the target channel/rank */ 2186 address = get_addr(ch, rk); 2187 2188 /* request HTE reconfiguration */ 2189 mrc_params->hte_setup = 1; 2190 2191 /* check the settings */ 2192 do { 2193 /* result[07:00] == failing byte lane (MAX 8) */ 2194 result = check_bls_ex(mrc_params, address); 2195 /* check for failures */ 2196 if (result & 0xff) { 2197 /* at least 1 byte lane failed */ 2198 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { 2199 if (result & 2200 (bl_mask << bl)) { 2201 if (side == L) 2202 delay[L][ch][rk][bl] += WDQ_STEP; 2203 else 2204 delay[R][ch][rk][bl] -= WDQ_STEP; 2205 2206 /* check for algorithm failure */ 2207 if (delay[L][ch][rk][bl] != delay[R][ch][rk][bl]) { 2208 /* 2209 * margin available 2210 * update delay setting 2211 */ 2212 set_wdq(ch, rk, bl, 2213 delay[side][ch][rk][bl]); 2214 } else { 2215 /* 2216 * no margin available 2217 * notify the user and halt 2218 */ 2219 training_message(ch, rk, bl); 2220 mrc_post_code(0xee, 0x80 + side); 2221 } 2222 } 2223 } 2224 } 2225 /* stop when all byte lanes pass */ 2226 } while (result & 0xff); 2227 } 2228 } 2229 } 2230 } 2231 } 2232 2233 /* program WDQ to the middle of passing window */ 2234 for (ch = 0; ch < NUM_CHANNELS; ch++) { 2235 if (mrc_params->channel_enables & (1 << ch)) { 2236 for (rk = 0; rk < NUM_RANKS; rk++) { 2237 if (mrc_params->rank_enables & (1 << rk)) { 2238#ifdef R2R_SHARING 2239 /* increment "num_ranks_enabled" */ 2240 num_ranks_enabled++; 2241#endif 2242 for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { 2243 DPF(D_INFO, 2244 "WDQ eye rank%d lane%d : %d-%d\n", 2245 rk, bl, 2246 delay[L][ch][rk][bl], 2247 delay[R][ch][rk][bl]); 2248 2249 temp = (delay[R][ch][rk][bl] + delay[L][ch][rk][bl]) / 2; 2250 2251#ifdef R2R_SHARING 2252 final_delay[ch][bl] += temp; 2253 set_wdq(ch, rk, bl, 2254 final_delay[ch][bl] / num_ranks_enabled); 2255#else 2256 set_wdq(ch, rk, bl, temp); 2257#endif 2258 } 2259 } 2260 } 2261 } 2262 } 2263#endif 2264 2265 LEAVEFN(); 2266} 2267 2268/* 2269 * This function will store relevant timing data 2270 * 2271 * This data will be used on subsequent boots to speed up boot times 2272 * and is required for Suspend To RAM capabilities. 2273 */ 2274void store_timings(struct mrc_params *mrc_params) 2275{ 2276 uint8_t ch, rk, bl; 2277 struct mrc_timings *mt = &mrc_params->timings; 2278 2279 for (ch = 0; ch < NUM_CHANNELS; ch++) { 2280 for (rk = 0; rk < NUM_RANKS; rk++) { 2281 for (bl = 0; bl < NUM_BYTE_LANES; bl++) { 2282 mt->rcvn[ch][rk][bl] = get_rcvn(ch, rk, bl); 2283 mt->rdqs[ch][rk][bl] = get_rdqs(ch, rk, bl); 2284 mt->wdqs[ch][rk][bl] = get_wdqs(ch, rk, bl); 2285 mt->wdq[ch][rk][bl] = get_wdq(ch, rk, bl); 2286 2287 if (rk == 0) 2288 mt->vref[ch][bl] = get_vref(ch, bl); 2289 } 2290 2291 mt->wctl[ch][rk] = get_wctl(ch, rk); 2292 } 2293 2294 mt->wcmd[ch] = get_wcmd(ch); 2295 } 2296 2297 /* need to save for a case of changing frequency after warm reset */ 2298 mt->ddr_speed = mrc_params->ddr_speed; 2299} 2300 2301/* 2302 * The purpose of this function is to ensure the SEC comes out of reset 2303 * and IA initiates the SEC enabling Memory Scrambling. 2304 */ 2305void enable_scrambling(struct mrc_params *mrc_params) 2306{ 2307 uint32_t lfsr = 0; 2308 uint8_t i; 2309 2310 if (mrc_params->scrambling_enables == 0) 2311 return; 2312 2313 ENTERFN(); 2314 2315 /* 32 bit seed is always stored in BIOS NVM */ 2316 lfsr = mrc_params->timings.scrambler_seed; 2317 2318 if (mrc_params->boot_mode == BM_COLD) { 2319 /* 2320 * factory value is 0 and in first boot, 2321 * a clock based seed is loaded. 2322 */ 2323 if (lfsr == 0) { 2324 /* 2325 * get seed from system clock 2326 * and make sure it is not all 1's 2327 */ 2328 lfsr = rdtsc() & 0x0fffffff; 2329 } else { 2330 /* 2331 * Need to replace scrambler 2332 * 2333 * get next 32bit LFSR 16 times which is the last 2334 * part of the previous scrambler vector 2335 */ 2336 for (i = 0; i < 16; i++) 2337 lfsr32(&lfsr); 2338 } 2339 2340 /* save new seed */ 2341 mrc_params->timings.scrambler_seed = lfsr; 2342 } 2343 2344 /* 2345 * In warm boot or S3 exit, we have the previous seed. 2346 * In cold boot, we have the last 32bit LFSR which is the new seed. 2347 */ 2348 lfsr32(&lfsr); /* shift to next value */ 2349 msg_port_write(MEM_CTLR, SCRMSEED, (lfsr & 0x0003ffff)); 2350 2351 for (i = 0; i < 2; i++) 2352 msg_port_write(MEM_CTLR, SCRMLO + i, (lfsr & 0xaaaaaaaa)); 2353 2354 LEAVEFN(); 2355} 2356 2357/* 2358 * Configure MCU Power Management Control Register 2359 * and Scheduler Control Register 2360 */ 2361void prog_ddr_control(struct mrc_params *mrc_params) 2362{ 2363 u32 dsch; 2364 u32 dpmc0; 2365 2366 ENTERFN(); 2367 2368 dsch = msg_port_read(MEM_CTLR, DSCH); 2369 dsch &= ~(DSCH_OOODIS | DSCH_OOOST3DIS | DSCH_NEWBYPDIS); 2370 msg_port_write(MEM_CTLR, DSCH, dsch); 2371 2372 dpmc0 = msg_port_read(MEM_CTLR, DPMC0); 2373 dpmc0 &= ~DPMC0_DISPWRDN; 2374 dpmc0 |= (mrc_params->power_down_disable << 25); 2375 dpmc0 &= ~DPMC0_CLKGTDIS; 2376 dpmc0 &= ~DPMC0_PCLSTO_MASK; 2377 dpmc0 |= (4 << 16); 2378 dpmc0 |= DPMC0_PREAPWDEN; 2379 msg_port_write(MEM_CTLR, DPMC0, dpmc0); 2380 2381 /* CMDTRIST = 2h - CMD/ADDR are tristated when no valid command */ 2382 mrc_write_mask(MEM_CTLR, DPMC1, 0x20, 0x30); 2383 2384 LEAVEFN(); 2385} 2386 2387/* 2388 * After training complete configure MCU Rank Population Register 2389 * specifying: ranks enabled, device width, density, address mode 2390 */ 2391void prog_dra_drb(struct mrc_params *mrc_params) 2392{ 2393 u32 drp; 2394 u32 dco; 2395 u8 density = mrc_params->params.density; 2396 2397 ENTERFN(); 2398 2399 dco = msg_port_read(MEM_CTLR, DCO); 2400 dco &= ~DCO_IC; 2401 msg_port_write(MEM_CTLR, DCO, dco); 2402 2403 drp = 0; 2404 if (mrc_params->rank_enables & 1) 2405 drp |= DRP_RKEN0; 2406 if (mrc_params->rank_enables & 2) 2407 drp |= DRP_RKEN1; 2408 if (mrc_params->dram_width == X16) { 2409 drp |= (1 << 4); 2410 drp |= (1 << 9); 2411 } 2412 2413 /* 2414 * Density encoding in struct dram_params: 0=512Mb, 1=Gb, 2=2Gb, 3=4Gb 2415 * has to be mapped RANKDENSx encoding (0=1Gb) 2416 */ 2417 if (density == 0) 2418 density = 4; 2419 2420 drp |= ((density - 1) << 6); 2421 drp |= ((density - 1) << 11); 2422 2423 /* Address mode can be overwritten if ECC enabled */ 2424 drp |= (mrc_params->address_mode << 14); 2425 2426 msg_port_write(MEM_CTLR, DRP, drp); 2427 2428 dco &= ~DCO_PMICTL; 2429 dco |= DCO_IC; 2430 msg_port_write(MEM_CTLR, DCO, dco); 2431 2432 LEAVEFN(); 2433} 2434 2435/* Send DRAM wake command */ 2436void perform_wake(struct mrc_params *mrc_params) 2437{ 2438 ENTERFN(); 2439 2440 dram_wake_command(); 2441 2442 LEAVEFN(); 2443} 2444 2445/* 2446 * Configure refresh rate and short ZQ calibration interval 2447 * Activate dynamic self refresh 2448 */ 2449void change_refresh_period(struct mrc_params *mrc_params) 2450{ 2451 u32 drfc; 2452 u32 dcal; 2453 u32 dpmc0; 2454 2455 ENTERFN(); 2456 2457 drfc = msg_port_read(MEM_CTLR, DRFC); 2458 drfc &= ~DRFC_TREFI_MASK; 2459 drfc |= (mrc_params->refresh_rate << 12); 2460 drfc |= DRFC_REFDBTCLR; 2461 msg_port_write(MEM_CTLR, DRFC, drfc); 2462 2463 dcal = msg_port_read(MEM_CTLR, DCAL); 2464 dcal &= ~DCAL_ZQCINT_MASK; 2465 dcal |= (3 << 8); /* 63ms */ 2466 msg_port_write(MEM_CTLR, DCAL, dcal); 2467 2468 dpmc0 = msg_port_read(MEM_CTLR, DPMC0); 2469 dpmc0 |= (DPMC0_DYNSREN | DPMC0_ENPHYCLKGATE); 2470 msg_port_write(MEM_CTLR, DPMC0, dpmc0); 2471 2472 LEAVEFN(); 2473} 2474 2475/* 2476 * Configure DDRPHY for Auto-Refresh, Periodic Compensations, 2477 * Dynamic Diff-Amp, ZQSPERIOD, Auto-Precharge, CKE Power-Down 2478 */ 2479void set_auto_refresh(struct mrc_params *mrc_params) 2480{ 2481 uint32_t channel; 2482 uint32_t rank; 2483 uint32_t bl; 2484 uint32_t bl_divisor = 1; 2485 uint32_t temp; 2486 2487 ENTERFN(); 2488 2489 /* 2490 * Enable Auto-Refresh, Periodic Compensations, Dynamic Diff-Amp, 2491 * ZQSPERIOD, Auto-Precharge, CKE Power-Down 2492 */ 2493 for (channel = 0; channel < NUM_CHANNELS; channel++) { 2494 if (mrc_params->channel_enables & (1 << channel)) { 2495 /* Enable Periodic RCOMPS */ 2496 mrc_alt_write_mask(DDRPHY, CMPCTRL, 2, 2); 2497 2498 /* Enable Dynamic DiffAmp & Set Read ODT Value */ 2499 switch (mrc_params->rd_odt_value) { 2500 case 0: 2501 temp = 0x3f; /* OFF */ 2502 break; 2503 default: 2504 temp = 0x00; /* Auto */ 2505 break; 2506 } 2507 2508 for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) { 2509 /* Override: DIFFAMP, ODT */ 2510 mrc_alt_write_mask(DDRPHY, 2511 B0OVRCTL + bl * DDRIODQ_BL_OFFSET + 2512 channel * DDRIODQ_CH_OFFSET, 2513 temp << 10, 2514 0x003ffc00); 2515 2516 /* Override: DIFFAMP, ODT */ 2517 mrc_alt_write_mask(DDRPHY, 2518 B1OVRCTL + bl * DDRIODQ_BL_OFFSET + 2519 channel * DDRIODQ_CH_OFFSET, 2520 temp << 10, 2521 0x003ffc00); 2522 } 2523 2524 /* Issue ZQCS command */ 2525 for (rank = 0; rank < NUM_RANKS; rank++) { 2526 if (mrc_params->rank_enables & (1 << rank)) 2527 dram_init_command(DCMD_ZQCS(rank)); 2528 } 2529 } 2530 } 2531 2532 clear_pointers(); 2533 2534 LEAVEFN(); 2535} 2536 2537/* 2538 * Depending on configuration enables ECC support 2539 * 2540 * Available memory size is decreased, and updated with 0s 2541 * in order to clear error status. Address mode 2 forced. 2542 */ 2543void ecc_enable(struct mrc_params *mrc_params) 2544{ 2545 u32 drp; 2546 u32 dsch; 2547 u32 ecc_ctrl; 2548 2549 if (mrc_params->ecc_enables == 0) 2550 return; 2551 2552 ENTERFN(); 2553 2554 /* Configuration required in ECC mode */ 2555 drp = msg_port_read(MEM_CTLR, DRP); 2556 drp &= ~DRP_ADDRMAP_MASK; 2557 drp |= DRP_ADDRMAP_MAP1; 2558 drp |= DRP_PRI64BSPLITEN; 2559 msg_port_write(MEM_CTLR, DRP, drp); 2560 2561 /* Disable new request bypass */ 2562 dsch = msg_port_read(MEM_CTLR, DSCH); 2563 dsch |= DSCH_NEWBYPDIS; 2564 msg_port_write(MEM_CTLR, DSCH, dsch); 2565 2566 /* Enable ECC */ 2567 ecc_ctrl = (DECCCTRL_SBEEN | DECCCTRL_DBEEN | DECCCTRL_ENCBGEN); 2568 msg_port_write(MEM_CTLR, DECCCTRL, ecc_ctrl); 2569 2570 /* Assume 8 bank memory, one bank is gone for ECC */ 2571 mrc_params->mem_size -= mrc_params->mem_size / 8; 2572 2573 /* For S3 resume memory content has to be preserved */ 2574 if (mrc_params->boot_mode != BM_S3) { 2575 select_hte(); 2576 hte_mem_init(mrc_params, MRC_MEM_INIT); 2577 select_mem_mgr(); 2578 } 2579 2580 LEAVEFN(); 2581} 2582 2583/* 2584 * Execute memory test 2585 * if error detected it is indicated in mrc_params->status 2586 */ 2587void memory_test(struct mrc_params *mrc_params) 2588{ 2589 uint32_t result = 0; 2590 2591 ENTERFN(); 2592 2593 select_hte(); 2594 result = hte_mem_init(mrc_params, MRC_MEM_TEST); 2595 select_mem_mgr(); 2596 2597 DPF(D_INFO, "Memory test result %x\n", result); 2598 mrc_params->status = ((result == 0) ? MRC_SUCCESS : MRC_E_MEMTEST); 2599 LEAVEFN(); 2600} 2601 2602/* Lock MCU registers at the end of initialization sequence */ 2603void lock_registers(struct mrc_params *mrc_params) 2604{ 2605 u32 dco; 2606 2607 ENTERFN(); 2608 2609 dco = msg_port_read(MEM_CTLR, DCO); 2610 dco &= ~(DCO_PMICTL | DCO_PMIDIS); 2611 dco |= (DCO_DRPLOCK | DCO_CPGCLOCK); 2612 msg_port_write(MEM_CTLR, DCO, dco); 2613 2614 LEAVEFN(); 2615} 2616