1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2020 MediaTek Inc. 4 * 5 * Author: Weijie Gao <weijie.gao@mediatek.com> 6 */ 7 8#include <asm/addrspace.h> 9#include <asm/cacheops.h> 10#include <asm/global_data.h> 11#include <linux/bitops.h> 12#include <linux/io.h> 13#include <mach/mc.h> 14 15DECLARE_GLOBAL_DATA_PTR; 16 17#define COARSE_MIN_START 6 18#define FINE_MIN_START 15 19#define COARSE_MAX_START 7 20#define FINE_MAX_START 0 21 22#define NUM_OF_CACHELINE 128 23#define TEST_PAT_SIZE (NUM_OF_CACHELINE * CONFIG_SYS_CACHELINE_SIZE) 24 25#define INIT_DQS_VAL ((7 << DQS1_DELAY_COARSE_TUNING_S) | \ 26 (4 << DQS1_DELAY_FINE_TUNING_S) | \ 27 (7 << DQS0_DELAY_COARSE_TUNING_S) | \ 28 (4 << DQS0_DELAY_FINE_TUNING_S)) 29 30static inline void pref_op(int op, const volatile void *addr) 31{ 32 __asm__ __volatile__("pref %0, 0(%1)" : : "i" (op), "r" (addr)); 33} 34 35static inline bool dqs_test_error(void __iomem *memc, u32 memsize, u32 dqsval, 36 u32 bias) 37{ 38 u32 *nca, *ca; 39 u32 off; 40 int i; 41 42 for (off = 0; off < memsize - TEST_PAT_SIZE; off += (memsize >> 6)) { 43 nca = (u32 *)KSEG1ADDR(off); 44 ca = (u32 *)KSEG0ADDR(off); 45 46 writel(INIT_DQS_VAL, memc + MEMCTL_DDR_DQS_DLY_REG); 47 wmb(); 48 49 for (i = 0; i < TEST_PAT_SIZE / sizeof(u32); i++) 50 ca[i] = 0x1f1f1f1f; 51 52 for (i = 0; i < TEST_PAT_SIZE / sizeof(u32); i++) 53 nca[i] = (u32)nca + i + bias; 54 55 writel(dqsval, memc + MEMCTL_DDR_DQS_DLY_REG); 56 wmb(); 57 58 for (i = 0; i < TEST_PAT_SIZE; i += CONFIG_SYS_CACHELINE_SIZE) 59 mips_cache(HIT_INVALIDATE_D, (u8 *)ca + i); 60 wmb(); 61 62 for (i = 0; i < TEST_PAT_SIZE; i += CONFIG_SYS_CACHELINE_SIZE) 63 pref_op(0, (u8 *)ca + i); 64 65 for (i = 0; i < TEST_PAT_SIZE / sizeof(u32); i++) { 66 if (ca[i] != (u32)nca + i + bias) 67 return true; 68 } 69 } 70 71 return false; 72} 73 74static inline int dqs_find_max(void __iomem *memc, u32 memsize, int initval, 75 int maxval, int shift, u32 regval) 76{ 77 int fieldval; 78 u32 dqsval; 79 80 for (fieldval = initval; fieldval <= maxval; fieldval++) { 81 dqsval = regval | (fieldval << shift); 82 if (dqs_test_error(memc, memsize, dqsval, 3)) 83 return max(fieldval - 1, initval); 84 } 85 86 return maxval; 87} 88 89static inline int dqs_find_min(void __iomem *memc, u32 memsize, int initval, 90 int minval, int shift, u32 regval) 91{ 92 int fieldval; 93 u32 dqsval; 94 95 for (fieldval = initval; fieldval >= minval; fieldval--) { 96 dqsval = regval | (fieldval << shift); 97 if (dqs_test_error(memc, memsize, dqsval, 1)) 98 return min(fieldval + 1, initval); 99 } 100 101 return minval; 102} 103 104void ddr_calibrate(void __iomem *memc, u32 memsize, u32 bw) 105{ 106 u32 dqs_coarse_min, dqs_coarse_max, dqs_coarse_val; 107 u32 dqs_fine_min, dqs_fine_max, dqs_fine_val; 108 u32 dqs_coarse_min_limit, dqs_fine_min_limit; 109 u32 dlls, dqs_dll, ddr_cfg2_reg; 110 u32 dqs_dly_tmp, dqs_dly, test_dqs, shift; 111 u32 rem, mask; 112 int i; 113 114 /* Disable Self-refresh */ 115 clrbits_32(memc + MEMCTL_DDR_SELF_REFRESH_REG, SR_AUTO_EN); 116 117 /* Save DDR_CFG2 and modify its DQS gating window */ 118 ddr_cfg2_reg = readl(memc + MEMCTL_DDR_CFG2_REG); 119 mask = DQS0_GATING_WINDOW_M; 120 if (bw == IND_SDRAM_WIDTH_16BIT) 121 mask |= DQS1_GATING_WINDOW_M; 122 clrbits_32(memc + MEMCTL_DDR_CFG2_REG, mask); 123 124 /* Get minimum available DQS value */ 125 dlls = readl(memc + MEMCTL_DLL_DBG_REG); 126 dlls = (dlls & MST_DLY_SEL_M) >> MST_DLY_SEL_S; 127 128 dqs_dll = dlls >> 4; 129 if (dqs_dll <= 8) 130 dqs_coarse_min_limit = 8 - dqs_dll; 131 else 132 dqs_coarse_min_limit = 0; 133 134 dqs_dll = dlls & 0xf; 135 if (dqs_dll <= 8) 136 dqs_fine_min_limit = 8 - dqs_dll; 137 else 138 dqs_fine_min_limit = 0; 139 140 /* Initial DQS register value */ 141 dqs_dly = INIT_DQS_VAL; 142 143 /* Calibrate DQS0 and/or DQS1 */ 144 for (i = 0; i < bw; i++) { 145 shift = i * 8; 146 dqs_dly &= ~(0xff << shift); 147 148 /* Find maximum DQS coarse-grain */ 149 dqs_dly_tmp = dqs_dly | (0xf << shift); 150 dqs_coarse_max = dqs_find_max(memc, memsize, COARSE_MAX_START, 151 0xf, 4 + shift, dqs_dly_tmp); 152 153 /* Find maximum DQS fine-grain */ 154 dqs_dly_tmp = dqs_dly | (dqs_coarse_max << (4 + shift)); 155 test_dqs = dqs_find_max(memc, memsize, FINE_MAX_START, 0xf, 156 shift, dqs_dly_tmp); 157 158 if (test_dqs == FINE_MAX_START) { 159 dqs_coarse_max--; 160 dqs_fine_max = 0xf; 161 } else { 162 dqs_fine_max = test_dqs - 1; 163 } 164 165 /* Find minimum DQS coarse-grain */ 166 dqs_dly_tmp = dqs_dly; 167 dqs_coarse_min = dqs_find_min(memc, memsize, COARSE_MIN_START, 168 dqs_coarse_min_limit, 4 + shift, 169 dqs_dly_tmp); 170 171 /* Find minimum DQS fine-grain */ 172 dqs_dly_tmp = dqs_dly | (dqs_coarse_min << (4 + shift)); 173 test_dqs = dqs_find_min(memc, memsize, FINE_MIN_START, 174 dqs_fine_min_limit, shift, dqs_dly_tmp); 175 176 if (test_dqs == FINE_MIN_START + 1) { 177 dqs_coarse_min++; 178 dqs_fine_min = 0; 179 } else { 180 dqs_fine_min = test_dqs; 181 } 182 183 /* Calculate central DQS coarse/fine value */ 184 dqs_coarse_val = (dqs_coarse_max + dqs_coarse_min) >> 1; 185 rem = (dqs_coarse_max + dqs_coarse_min) % 2; 186 187 dqs_fine_val = (rem * 4) + ((dqs_fine_max + dqs_fine_min) >> 1); 188 if (dqs_fine_val >= 0x10) { 189 dqs_coarse_val++; 190 dqs_fine_val -= 8; 191 } 192 193 /* Save current DQS value */ 194 dqs_dly |= ((dqs_coarse_val << 4) | dqs_fine_val) << shift; 195 } 196 197 /* Set final DQS value */ 198 writel(dqs_dly, memc + MEMCTL_DDR_DQS_DLY_REG); 199 200 /* Restore DDR_CFG2 */ 201 writel(ddr_cfg2_reg, memc + MEMCTL_DDR_CFG2_REG); 202 203 /* Enable Self-refresh */ 204 setbits_32(memc + MEMCTL_DDR_SELF_REFRESH_REG, SR_AUTO_EN); 205} 206