1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright 2024 Google LLC. 4 */ 5#include <kunit/test.h> 6#include <linux/io-pgtable.h> 7 8#include "arm-smmu-v3.h" 9 10struct arm_smmu_test_writer { 11 struct arm_smmu_entry_writer writer; 12 struct kunit *test; 13 const __le64 *init_entry; 14 const __le64 *target_entry; 15 __le64 *entry; 16 17 bool invalid_entry_written; 18 unsigned int num_syncs; 19}; 20 21#define NUM_ENTRY_QWORDS 8 22#define NUM_EXPECTED_SYNCS(x) x 23 24static struct arm_smmu_ste bypass_ste; 25static struct arm_smmu_ste abort_ste; 26static struct arm_smmu_device smmu = { 27 .features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR 28}; 29static struct mm_struct sva_mm = { 30 .pgd = (void *)0xdaedbeefdeadbeefULL, 31}; 32 33static bool arm_smmu_entry_differs_in_used_bits(const __le64 *entry, 34 const __le64 *used_bits, 35 const __le64 *target, 36 unsigned int length) 37{ 38 bool differs = false; 39 unsigned int i; 40 41 for (i = 0; i < length; i++) { 42 if ((entry[i] & used_bits[i]) != target[i]) 43 differs = true; 44 } 45 return differs; 46} 47 48static void 49arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) 50{ 51 struct arm_smmu_test_writer *test_writer = 52 container_of(writer, struct arm_smmu_test_writer, writer); 53 __le64 *entry_used_bits; 54 55 entry_used_bits = kunit_kzalloc( 56 test_writer->test, sizeof(*entry_used_bits) * NUM_ENTRY_QWORDS, 57 GFP_KERNEL); 58 KUNIT_ASSERT_NOT_NULL(test_writer->test, entry_used_bits); 59 60 pr_debug("STE value is now set to: "); 61 print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, 62 test_writer->entry, 63 NUM_ENTRY_QWORDS * sizeof(*test_writer->entry), 64 false); 65 66 test_writer->num_syncs += 1; 67 if (!test_writer->entry[0]) { 68 test_writer->invalid_entry_written = true; 69 } else { 70 /* 71 * At any stage in a hitless transition, the entry must be 72 * equivalent to either the initial entry or the target entry 73 * when only considering the bits used by the current 74 * configuration. 75 */ 76 writer->ops->get_used(test_writer->entry, entry_used_bits); 77 KUNIT_EXPECT_FALSE( 78 test_writer->test, 79 arm_smmu_entry_differs_in_used_bits( 80 test_writer->entry, entry_used_bits, 81 test_writer->init_entry, NUM_ENTRY_QWORDS) && 82 arm_smmu_entry_differs_in_used_bits( 83 test_writer->entry, entry_used_bits, 84 test_writer->target_entry, 85 NUM_ENTRY_QWORDS)); 86 } 87} 88 89static void 90arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, 91 const __le64 *ste) 92{ 93 __le64 used_bits[NUM_ENTRY_QWORDS] = {}; 94 95 arm_smmu_get_ste_used(ste, used_bits); 96 pr_debug("STE used bits: "); 97 print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, used_bits, 98 sizeof(used_bits), false); 99} 100 101static const struct arm_smmu_entry_writer_ops test_ste_ops = { 102 .sync = arm_smmu_test_writer_record_syncs, 103 .get_used = arm_smmu_get_ste_used, 104}; 105 106static const struct arm_smmu_entry_writer_ops test_cd_ops = { 107 .sync = arm_smmu_test_writer_record_syncs, 108 .get_used = arm_smmu_get_cd_used, 109}; 110 111static void arm_smmu_v3_test_ste_expect_transition( 112 struct kunit *test, const struct arm_smmu_ste *cur, 113 const struct arm_smmu_ste *target, unsigned int num_syncs_expected, 114 bool hitless) 115{ 116 struct arm_smmu_ste cur_copy = *cur; 117 struct arm_smmu_test_writer test_writer = { 118 .writer = { 119 .ops = &test_ste_ops, 120 }, 121 .test = test, 122 .init_entry = cur->data, 123 .target_entry = target->data, 124 .entry = cur_copy.data, 125 .num_syncs = 0, 126 .invalid_entry_written = false, 127 128 }; 129 130 pr_debug("STE initial value: "); 131 print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, cur_copy.data, 132 sizeof(cur_copy), false); 133 arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, cur->data); 134 pr_debug("STE target value: "); 135 print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, target->data, 136 sizeof(cur_copy), false); 137 arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, 138 target->data); 139 140 arm_smmu_write_entry(&test_writer.writer, cur_copy.data, target->data); 141 142 KUNIT_EXPECT_EQ(test, test_writer.invalid_entry_written, !hitless); 143 KUNIT_EXPECT_EQ(test, test_writer.num_syncs, num_syncs_expected); 144 KUNIT_EXPECT_MEMEQ(test, target->data, cur_copy.data, sizeof(cur_copy)); 145} 146 147static void arm_smmu_v3_test_ste_expect_hitless_transition( 148 struct kunit *test, const struct arm_smmu_ste *cur, 149 const struct arm_smmu_ste *target, unsigned int num_syncs_expected) 150{ 151 arm_smmu_v3_test_ste_expect_transition(test, cur, target, 152 num_syncs_expected, true); 153} 154 155static const dma_addr_t fake_cdtab_dma_addr = 0xF0F0F0F0F0F0; 156 157static void arm_smmu_test_make_cdtable_ste(struct arm_smmu_ste *ste, 158 const dma_addr_t dma_addr) 159{ 160 struct arm_smmu_master master = { 161 .cd_table.cdtab_dma = dma_addr, 162 .cd_table.s1cdmax = 0xFF, 163 .cd_table.s1fmt = STRTAB_STE_0_S1FMT_64K_L2, 164 .smmu = &smmu, 165 }; 166 167 arm_smmu_make_cdtable_ste(ste, &master); 168} 169 170static void arm_smmu_v3_write_ste_test_bypass_to_abort(struct kunit *test) 171{ 172 /* 173 * Bypass STEs has used bits in the first two Qwords, while abort STEs 174 * only have used bits in the first QWord. Transitioning from bypass to 175 * abort requires two syncs: the first to set the first qword and make 176 * the STE into an abort, the second to clean up the second qword. 177 */ 178 arm_smmu_v3_test_ste_expect_hitless_transition( 179 test, &bypass_ste, &abort_ste, NUM_EXPECTED_SYNCS(2)); 180} 181 182static void arm_smmu_v3_write_ste_test_abort_to_bypass(struct kunit *test) 183{ 184 /* 185 * Transitioning from abort to bypass also requires two syncs: the first 186 * to set the second qword data required by the bypass STE, and the 187 * second to set the first qword and switch to bypass. 188 */ 189 arm_smmu_v3_test_ste_expect_hitless_transition( 190 test, &abort_ste, &bypass_ste, NUM_EXPECTED_SYNCS(2)); 191} 192 193static void arm_smmu_v3_write_ste_test_cdtable_to_abort(struct kunit *test) 194{ 195 struct arm_smmu_ste ste; 196 197 arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); 198 arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &abort_ste, 199 NUM_EXPECTED_SYNCS(2)); 200} 201 202static void arm_smmu_v3_write_ste_test_abort_to_cdtable(struct kunit *test) 203{ 204 struct arm_smmu_ste ste; 205 206 arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); 207 arm_smmu_v3_test_ste_expect_hitless_transition(test, &abort_ste, &ste, 208 NUM_EXPECTED_SYNCS(2)); 209} 210 211static void arm_smmu_v3_write_ste_test_cdtable_to_bypass(struct kunit *test) 212{ 213 struct arm_smmu_ste ste; 214 215 arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); 216 arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &bypass_ste, 217 NUM_EXPECTED_SYNCS(3)); 218} 219 220static void arm_smmu_v3_write_ste_test_bypass_to_cdtable(struct kunit *test) 221{ 222 struct arm_smmu_ste ste; 223 224 arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); 225 arm_smmu_v3_test_ste_expect_hitless_transition(test, &bypass_ste, &ste, 226 NUM_EXPECTED_SYNCS(3)); 227} 228 229static void arm_smmu_test_make_s2_ste(struct arm_smmu_ste *ste, 230 bool ats_enabled) 231{ 232 struct arm_smmu_master master = { 233 .smmu = &smmu, 234 .ats_enabled = ats_enabled, 235 }; 236 struct io_pgtable io_pgtable = {}; 237 struct arm_smmu_domain smmu_domain = { 238 .pgtbl_ops = &io_pgtable.ops, 239 }; 240 241 io_pgtable.cfg.arm_lpae_s2_cfg.vttbr = 0xdaedbeefdeadbeefULL; 242 io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.ps = 1; 243 io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.tg = 2; 244 io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.sh = 3; 245 io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.orgn = 1; 246 io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.irgn = 2; 247 io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.sl = 3; 248 io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.tsz = 4; 249 250 arm_smmu_make_s2_domain_ste(ste, &master, &smmu_domain); 251} 252 253static void arm_smmu_v3_write_ste_test_s2_to_abort(struct kunit *test) 254{ 255 struct arm_smmu_ste ste; 256 257 arm_smmu_test_make_s2_ste(&ste, true); 258 arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &abort_ste, 259 NUM_EXPECTED_SYNCS(2)); 260} 261 262static void arm_smmu_v3_write_ste_test_abort_to_s2(struct kunit *test) 263{ 264 struct arm_smmu_ste ste; 265 266 arm_smmu_test_make_s2_ste(&ste, true); 267 arm_smmu_v3_test_ste_expect_hitless_transition(test, &abort_ste, &ste, 268 NUM_EXPECTED_SYNCS(2)); 269} 270 271static void arm_smmu_v3_write_ste_test_s2_to_bypass(struct kunit *test) 272{ 273 struct arm_smmu_ste ste; 274 275 arm_smmu_test_make_s2_ste(&ste, true); 276 arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &bypass_ste, 277 NUM_EXPECTED_SYNCS(2)); 278} 279 280static void arm_smmu_v3_write_ste_test_bypass_to_s2(struct kunit *test) 281{ 282 struct arm_smmu_ste ste; 283 284 arm_smmu_test_make_s2_ste(&ste, true); 285 arm_smmu_v3_test_ste_expect_hitless_transition(test, &bypass_ste, &ste, 286 NUM_EXPECTED_SYNCS(2)); 287} 288 289static void arm_smmu_v3_test_cd_expect_transition( 290 struct kunit *test, const struct arm_smmu_cd *cur, 291 const struct arm_smmu_cd *target, unsigned int num_syncs_expected, 292 bool hitless) 293{ 294 struct arm_smmu_cd cur_copy = *cur; 295 struct arm_smmu_test_writer test_writer = { 296 .writer = { 297 .ops = &test_cd_ops, 298 }, 299 .test = test, 300 .init_entry = cur->data, 301 .target_entry = target->data, 302 .entry = cur_copy.data, 303 .num_syncs = 0, 304 .invalid_entry_written = false, 305 306 }; 307 308 pr_debug("CD initial value: "); 309 print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, cur_copy.data, 310 sizeof(cur_copy), false); 311 arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, cur->data); 312 pr_debug("CD target value: "); 313 print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, target->data, 314 sizeof(cur_copy), false); 315 arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, 316 target->data); 317 318 arm_smmu_write_entry(&test_writer.writer, cur_copy.data, target->data); 319 320 KUNIT_EXPECT_EQ(test, test_writer.invalid_entry_written, !hitless); 321 KUNIT_EXPECT_EQ(test, test_writer.num_syncs, num_syncs_expected); 322 KUNIT_EXPECT_MEMEQ(test, target->data, cur_copy.data, sizeof(cur_copy)); 323} 324 325static void arm_smmu_v3_test_cd_expect_non_hitless_transition( 326 struct kunit *test, const struct arm_smmu_cd *cur, 327 const struct arm_smmu_cd *target, unsigned int num_syncs_expected) 328{ 329 arm_smmu_v3_test_cd_expect_transition(test, cur, target, 330 num_syncs_expected, false); 331} 332 333static void arm_smmu_v3_test_cd_expect_hitless_transition( 334 struct kunit *test, const struct arm_smmu_cd *cur, 335 const struct arm_smmu_cd *target, unsigned int num_syncs_expected) 336{ 337 arm_smmu_v3_test_cd_expect_transition(test, cur, target, 338 num_syncs_expected, true); 339} 340 341static void arm_smmu_test_make_s1_cd(struct arm_smmu_cd *cd, unsigned int asid) 342{ 343 struct arm_smmu_master master = { 344 .smmu = &smmu, 345 }; 346 struct io_pgtable io_pgtable = {}; 347 struct arm_smmu_domain smmu_domain = { 348 .pgtbl_ops = &io_pgtable.ops, 349 .cd = { 350 .asid = asid, 351 }, 352 }; 353 354 io_pgtable.cfg.arm_lpae_s1_cfg.ttbr = 0xdaedbeefdeadbeefULL; 355 io_pgtable.cfg.arm_lpae_s1_cfg.tcr.ips = 1; 356 io_pgtable.cfg.arm_lpae_s1_cfg.tcr.tg = 2; 357 io_pgtable.cfg.arm_lpae_s1_cfg.tcr.sh = 3; 358 io_pgtable.cfg.arm_lpae_s1_cfg.tcr.orgn = 1; 359 io_pgtable.cfg.arm_lpae_s1_cfg.tcr.irgn = 2; 360 io_pgtable.cfg.arm_lpae_s1_cfg.tcr.tsz = 4; 361 io_pgtable.cfg.arm_lpae_s1_cfg.mair = 0xabcdef012345678ULL; 362 363 arm_smmu_make_s1_cd(cd, &master, &smmu_domain); 364} 365 366static void arm_smmu_v3_write_cd_test_s1_clear(struct kunit *test) 367{ 368 struct arm_smmu_cd cd = {}; 369 struct arm_smmu_cd cd_2; 370 371 arm_smmu_test_make_s1_cd(&cd_2, 1997); 372 arm_smmu_v3_test_cd_expect_non_hitless_transition( 373 test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); 374 arm_smmu_v3_test_cd_expect_non_hitless_transition( 375 test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); 376} 377 378static void arm_smmu_v3_write_cd_test_s1_change_asid(struct kunit *test) 379{ 380 struct arm_smmu_cd cd = {}; 381 struct arm_smmu_cd cd_2; 382 383 arm_smmu_test_make_s1_cd(&cd, 778); 384 arm_smmu_test_make_s1_cd(&cd_2, 1997); 385 arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd, &cd_2, 386 NUM_EXPECTED_SYNCS(1)); 387 arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd_2, &cd, 388 NUM_EXPECTED_SYNCS(1)); 389} 390 391static void arm_smmu_test_make_sva_cd(struct arm_smmu_cd *cd, unsigned int asid) 392{ 393 struct arm_smmu_master master = { 394 .smmu = &smmu, 395 }; 396 397 arm_smmu_make_sva_cd(cd, &master, &sva_mm, asid); 398} 399 400static void arm_smmu_test_make_sva_release_cd(struct arm_smmu_cd *cd, 401 unsigned int asid) 402{ 403 struct arm_smmu_master master = { 404 .smmu = &smmu, 405 }; 406 407 arm_smmu_make_sva_cd(cd, &master, NULL, asid); 408} 409 410static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test) 411{ 412 struct arm_smmu_cd cd = {}; 413 struct arm_smmu_cd cd_2; 414 415 arm_smmu_test_make_sva_cd(&cd_2, 1997); 416 arm_smmu_v3_test_cd_expect_non_hitless_transition( 417 test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); 418 arm_smmu_v3_test_cd_expect_non_hitless_transition( 419 test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); 420} 421 422static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test) 423{ 424 struct arm_smmu_cd cd; 425 struct arm_smmu_cd cd_2; 426 427 arm_smmu_test_make_sva_cd(&cd, 1997); 428 arm_smmu_test_make_sva_release_cd(&cd_2, 1997); 429 arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd, &cd_2, 430 NUM_EXPECTED_SYNCS(2)); 431 arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd_2, &cd, 432 NUM_EXPECTED_SYNCS(2)); 433} 434 435static struct kunit_case arm_smmu_v3_test_cases[] = { 436 KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_abort), 437 KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_bypass), 438 KUNIT_CASE(arm_smmu_v3_write_ste_test_cdtable_to_abort), 439 KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_cdtable), 440 KUNIT_CASE(arm_smmu_v3_write_ste_test_cdtable_to_bypass), 441 KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_cdtable), 442 KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_abort), 443 KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_s2), 444 KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_bypass), 445 KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_s2), 446 KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_clear), 447 KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_change_asid), 448 KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear), 449 KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release), 450 {}, 451}; 452 453static int arm_smmu_v3_test_suite_init(struct kunit_suite *test) 454{ 455 arm_smmu_make_bypass_ste(&smmu, &bypass_ste); 456 arm_smmu_make_abort_ste(&abort_ste); 457 return 0; 458} 459 460static struct kunit_suite arm_smmu_v3_test_module = { 461 .name = "arm-smmu-v3-kunit-test", 462 .suite_init = arm_smmu_v3_test_suite_init, 463 .test_cases = arm_smmu_v3_test_cases, 464}; 465kunit_test_suites(&arm_smmu_v3_test_module); 466 467MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING); 468MODULE_LICENSE("GPL v2"); 469