amd/atl/dehash.c

22 static inline bool map_bits_valid(struct addr_ctx *ctx, u8 bit1, u8 bit2,
25 	if (!(ctx->map.intlv_bit_pos == bit1 || ctx->map.intlv_bit_pos == bit2)) {
26 		pr_debug("Invalid interleave bit: %u", ctx->map.intlv_bit_pos);
30 	if (ctx->map.num_intlv_dies > num_intlv_dies) {
31 		pr_debug("Invalid number of interleave dies: %u", ctx->map.num_intlv_dies);
35 	if (ctx->map.num_intlv_sockets > num_intlv_sockets) {
36 		pr_debug("Invalid number of interleave sockets: %u", ctx->map.num_intlv_sockets);
43 static int df2_dehash_addr(struct addr_ctx *ctx)
47 	if (!map_bits_valid(ctx, 8, 9, 1, 1))
50 	intlv_bit_pos = ctx->map.intlv_bit_pos;
51 	intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr);
54 	hashed_bit ^= FIELD_GET(BIT_ULL(12), ctx->ret_addr);
55 	hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr);
56 	hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr);
57 	hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr);
60 		ctx->ret_addr ^= BIT_ULL(intlv_bit_pos);
65 static int df3_dehash_addr(struct addr_ctx *ctx)
70 	if (!map_bits_valid(ctx, 8, 9, 1, 1))
73 	hash_ctl_64k = FIELD_GET(DF3_HASH_CTL_64K, ctx->map.ctl);
74 	hash_ctl_2M  = FIELD_GET(DF3_HASH_CTL_2M, ctx->map.ctl);
75 	hash_ctl_1G  = FIELD_GET(DF3_HASH_CTL_1G, ctx->map.ctl);
77 	intlv_bit_pos = ctx->map.intlv_bit_pos;
78 	intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr);
81 	hashed_bit ^= FIELD_GET(BIT_ULL(14), ctx->ret_addr);
82 	hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k;
83 	hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M;
84 	hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G;
87 		ctx->ret_addr ^= BIT_ULL(intlv_bit_pos);
90 	if (ctx->map.intlv_mode == DF3_COD4_2CHAN_HASH)
93 	intlv_bit = FIELD_GET(BIT_ULL(12), ctx->ret_addr);
96 	hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k;
97 	hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M;
98 	hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G;
101 		ctx->ret_addr ^= BIT_ULL(12);
104 	if (ctx->map.intlv_mode == DF3_COD2_4CHAN_HASH)
107 	intlv_bit = FIELD_GET(BIT_ULL(13), ctx->ret_addr);
110 	hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k;
111 	hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M;
112 	hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G;
115 		ctx->ret_addr ^= BIT_ULL(13);
120 static int df3_6chan_dehash_addr(struct addr_ctx *ctx)
122 	u8 intlv_bit_pos = ctx->map.intlv_bit_pos;
126 	if (ctx->map.intlv_mode != DF3_6CHAN) {
127 		atl_debug_on_bad_intlv_mode(ctx);
131 	num_intlv_bits = ilog2(ctx->map.num_intlv_chan) + 1;
133 	hash_ctl_2M = FIELD_GET(DF3_HASH_CTL_2M, ctx->map.ctl);
134 	hash_ctl_1G = FIELD_GET(DF3_HASH_CTL_1G, ctx->map.ctl);
136 	intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr);
139 	hashed_bit ^= !!(BIT_ULL(intlv_bit_pos + num_intlv_bits) & ctx->ret_addr);
140 	hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M;
141 	hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G;
144 		ctx->ret_addr ^= BIT_ULL(intlv_bit_pos);
147 	intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr);
150 	hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M;
151 	hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G;
154 		ctx->ret_addr ^= BIT_ULL(intlv_bit_pos);
157 	intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr);
160 	hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M;
161 	hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G;
164 		ctx->ret_addr ^= BIT_ULL(intlv_bit_pos);
169 static int df4_dehash_addr(struct addr_ctx *ctx)
174 	if (!map_bits_valid(ctx, 8, 8, 1, 2))
177 	hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl);
178 	hash_ctl_2M  = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl);
179 	hash_ctl_1G  = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl);
181 	intlv_bit = FIELD_GET(BIT_ULL(8), ctx->ret_addr);
184 	hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k;
185 	hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M;
186 	hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G;
188 	if (ctx->map.num_intlv_sockets == 1)
189 		hashed_bit ^= FIELD_GET(BIT_ULL(14), ctx->ret_addr);
192 		ctx->ret_addr ^= BIT_ULL(8);
200 	if (ctx->map.total_intlv_chan <= 2)
203 	intlv_bit = FIELD_GET(BIT_ULL(12), ctx->ret_addr);
206 	hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k;
207 	hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M;
208 	hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G;
211 		ctx->ret_addr ^= BIT_ULL(12);
214 	if (ctx->map.total_intlv_chan <= 4)
217 	intlv_bit = FIELD_GET(BIT_ULL(13), ctx->ret_addr);
220 	hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k;
221 	hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M;
222 	hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G;
225 		ctx->ret_addr ^= BIT_ULL(13);
228 	if (ctx->map.total_intlv_chan <= 8)
231 	intlv_bit = FIELD_GET(BIT_ULL(14), ctx->ret_addr);
234 	hashed_bit ^= FIELD_GET(BIT_ULL(19), ctx->ret_addr) & hash_ctl_64k;
235 	hashed_bit ^= FIELD_GET(BIT_ULL(24), ctx->ret_addr) & hash_ctl_2M;
236 	hashed_bit ^= FIELD_GET(BIT_ULL(33), ctx->ret_addr) & hash_ctl_1G;
239 		ctx->ret_addr ^= BIT_ULL(14);
244 static int df4p5_dehash_addr(struct addr_ctx *ctx)
250 	if (!map_bits_valid(ctx, 8, 8, 1, 2))
253 	hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl);
254 	hash_ctl_2M  = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl);
255 	hash_ctl_1G  = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl);
256 	hash_ctl_1T  = FIELD_GET(DF4p5_HASH_CTL_1T, ctx->map.ctl);
268 	rehash_vector = ctx->map.total_intlv_chan - 1;
271 	if (ctx->map.intlv_mode == DF4p5_NPS2_4CHAN_1K_HASH ||
272 	    ctx->map.intlv_mode == DF4p5_NPS1_8CHAN_1K_HASH ||
273 	    ctx->map.intlv_mode == DF4p5_NPS1_16CHAN_1K_HASH)
279 		intlv_bit = FIELD_GET(BIT_ULL(8), ctx->ret_addr);
282 		hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k;
283 		hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M;
284 		hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G;
285 		hashed_bit ^= FIELD_GET(BIT_ULL(40), ctx->ret_addr) & hash_ctl_1T;
288 			ctx->ret_addr ^= BIT_ULL(8);
292 		intlv_bit = FIELD_GET(BIT_ULL(9), ctx->ret_addr);
295 		hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k;
296 		hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M;
297 		hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G;
298 		hashed_bit ^= FIELD_GET(BIT_ULL(41), ctx->ret_addr) & hash_ctl_1T;
301 			ctx->ret_addr ^= BIT_ULL(9);
305 		intlv_bit = FIELD_GET(BIT_ULL(12), ctx->ret_addr);
308 		hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k;
309 		hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M;
310 		hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G;
311 		hashed_bit ^= FIELD_GET(BIT_ULL(42), ctx->ret_addr) & hash_ctl_1T;
314 			ctx->ret_addr ^= BIT_ULL(12);
318 		intlv_bit = FIELD_GET(BIT_ULL(13), ctx->ret_addr);
321 		hashed_bit ^= FIELD_GET(BIT_ULL(19), ctx->ret_addr) & hash_ctl_64k;
322 		hashed_bit ^= FIELD_GET(BIT_ULL(24), ctx->ret_addr) & hash_ctl_2M;
323 		hashed_bit ^= FIELD_GET(BIT_ULL(33), ctx->ret_addr) & hash_ctl_1G;
324 		hashed_bit ^= FIELD_GET(BIT_ULL(43), ctx->ret_addr) & hash_ctl_1T;
327 			ctx->ret_addr ^= BIT_ULL(13);
331 		intlv_bit = FIELD_GET(BIT_ULL(14), ctx->ret_addr);
334 		hashed_bit ^= FIELD_GET(BIT_ULL(20), ctx->ret_addr) & hash_ctl_64k;
335 		hashed_bit ^= FIELD_GET(BIT_ULL(25), ctx->ret_addr) & hash_ctl_2M;
336 		hashed_bit ^= FIELD_GET(BIT_ULL(34), ctx->ret_addr) & hash_ctl_1G;
337 		hashed_bit ^= FIELD_GET(BIT_ULL(44), ctx->ret_addr) & hash_ctl_1T;
340 			ctx->ret_addr ^= BIT_ULL(14);
357 static int mi300_dehash_addr(struct addr_ctx *ctx)
363 	if (!map_bits_valid(ctx, 8, 8, 4, 1))
366 	hash_ctl_4k  = FIELD_GET(DF4p5_HASH_CTL_4K, ctx->map.ctl);
367 	hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K,  ctx->map.ctl);
368 	hash_ctl_2M  = FIELD_GET(DF4_HASH_CTL_2M,   ctx->map.ctl);
369 	hash_ctl_1G  = FIELD_GET(DF4_HASH_CTL_1G,   ctx->map.ctl);
370 	hash_ctl_1T  = FIELD_GET(DF4p5_HASH_CTL_1T, ctx->map.ctl);
373 	num_intlv_bits = ilog2(ctx->map.num_intlv_chan);
382 		intlv_bit = BIT_ULL(base_bit) & ctx->ret_addr;
388 			test_bit    = BIT_ULL(12 + i) & ctx->ret_addr;
393 		test_bit    = BIT_ULL(15 + i) & ctx->ret_addr;
395 		test_bit    = BIT_ULL(22 + i) & ctx->ret_addr;
397 		test_bit    = BIT_ULL(29 + i) & ctx->ret_addr;
399 		test_bit    = BIT_ULL(36 + i) & ctx->ret_addr;
401 		test_bit    = BIT_ULL(43 + i) & ctx->ret_addr;
405 			ctx->ret_addr ^= BIT_ULL(base_bit);
409 	num_intlv_bits = ilog2(ctx->map.num_intlv_dies);
414 		intlv_bit = BIT_ULL(base_bit) & ctx->ret_addr;
418 		test_bit    = BIT_ULL(20 + i) & ctx->ret_addr;
420 		test_bit    = BIT_ULL(27 + i) & ctx->ret_addr;
422 		test_bit    = BIT_ULL(34 + i) & ctx->ret_addr;
424 		test_bit    = BIT_ULL(41 + i) & ctx->ret_addr;
428 			ctx->ret_addr ^= BIT_ULL(base_bit);
434 int dehash_address(struct addr_ctx *ctx)
436 	switch (ctx->map.intlv_mode) {
466 		return df2_dehash_addr(ctx);
471 		return df3_dehash_addr(ctx);
474 		return df3_6chan_dehash_addr(ctx);
479 		return df4_dehash_addr(ctx);
489 		return df4p5_dehash_addr(ctx);
494 		return mi300_dehash_addr(ctx);
497 		atl_debug_on_bad_intlv_mode(ctx);