1// SPDX-License-Identifier: GPL-2.0 2/* 3 * t10_pi.c - Functions for generating and verifying T10 Protection 4 * Information. 5 */ 6 7#include <linux/t10-pi.h> 8#include <linux/blk-integrity.h> 9#include <linux/crc-t10dif.h> 10#include <linux/crc64.h> 11#include <linux/module.h> 12#include <net/checksum.h> 13#include <asm/unaligned.h> 14 15typedef __be16 (csum_fn) (__be16, void *, unsigned int); 16 17static __be16 t10_pi_crc_fn(__be16 crc, void *data, unsigned int len) 18{ 19 return cpu_to_be16(crc_t10dif_update(be16_to_cpu(crc), data, len)); 20} 21 22static __be16 t10_pi_ip_fn(__be16 csum, void *data, unsigned int len) 23{ 24 return (__force __be16)ip_compute_csum(data, len); 25} 26 27/* 28 * Type 1 and Type 2 protection use the same format: 16 bit guard tag, 29 * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref 30 * tag. 31 */ 32static blk_status_t t10_pi_generate(struct blk_integrity_iter *iter, 33 csum_fn *fn, enum t10_dif_type type) 34{ 35 u8 offset = iter->pi_offset; 36 unsigned int i; 37 38 for (i = 0 ; i < iter->data_size ; i += iter->interval) { 39 struct t10_pi_tuple *pi = iter->prot_buf + offset; 40 41 pi->guard_tag = fn(0, iter->data_buf, iter->interval); 42 if (offset) 43 pi->guard_tag = fn(pi->guard_tag, iter->prot_buf, 44 offset); 45 pi->app_tag = 0; 46 47 if (type == T10_PI_TYPE1_PROTECTION) 48 pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed)); 49 else 50 pi->ref_tag = 0; 51 52 iter->data_buf += iter->interval; 53 iter->prot_buf += iter->tuple_size; 54 iter->seed++; 55 } 56 57 return BLK_STS_OK; 58} 59 60static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter, 61 csum_fn *fn, enum t10_dif_type type) 62{ 63 u8 offset = iter->pi_offset; 64 unsigned int i; 65 66 BUG_ON(type == T10_PI_TYPE0_PROTECTION); 67 68 for (i = 0 ; i < iter->data_size ; i += iter->interval) { 69 struct t10_pi_tuple *pi = iter->prot_buf + offset; 70 __be16 csum; 71 72 if (type == T10_PI_TYPE1_PROTECTION || 73 type == T10_PI_TYPE2_PROTECTION) { 74 if (pi->app_tag == T10_PI_APP_ESCAPE) 75 goto next; 76 77 if (be32_to_cpu(pi->ref_tag) != 78 lower_32_bits(iter->seed)) { 79 pr_err("%s: ref tag error at location %llu " \ 80 "(rcvd %u)\n", iter->disk_name, 81 (unsigned long long) 82 iter->seed, be32_to_cpu(pi->ref_tag)); 83 return BLK_STS_PROTECTION; 84 } 85 } else if (type == T10_PI_TYPE3_PROTECTION) { 86 if (pi->app_tag == T10_PI_APP_ESCAPE && 87 pi->ref_tag == T10_PI_REF_ESCAPE) 88 goto next; 89 } 90 91 csum = fn(0, iter->data_buf, iter->interval); 92 if (offset) 93 csum = fn(csum, iter->prot_buf, offset); 94 95 if (pi->guard_tag != csum) { 96 pr_err("%s: guard tag error at sector %llu " \ 97 "(rcvd %04x, want %04x)\n", iter->disk_name, 98 (unsigned long long)iter->seed, 99 be16_to_cpu(pi->guard_tag), be16_to_cpu(csum)); 100 return BLK_STS_PROTECTION; 101 } 102 103next: 104 iter->data_buf += iter->interval; 105 iter->prot_buf += iter->tuple_size; 106 iter->seed++; 107 } 108 109 return BLK_STS_OK; 110} 111 112static blk_status_t t10_pi_type1_generate_crc(struct blk_integrity_iter *iter) 113{ 114 return t10_pi_generate(iter, t10_pi_crc_fn, T10_PI_TYPE1_PROTECTION); 115} 116 117static blk_status_t t10_pi_type1_generate_ip(struct blk_integrity_iter *iter) 118{ 119 return t10_pi_generate(iter, t10_pi_ip_fn, T10_PI_TYPE1_PROTECTION); 120} 121 122static blk_status_t t10_pi_type1_verify_crc(struct blk_integrity_iter *iter) 123{ 124 return t10_pi_verify(iter, t10_pi_crc_fn, T10_PI_TYPE1_PROTECTION); 125} 126 127static blk_status_t t10_pi_type1_verify_ip(struct blk_integrity_iter *iter) 128{ 129 return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE1_PROTECTION); 130} 131 132/** 133 * t10_pi_type1_prepare - prepare PI prior submitting request to device 134 * @rq: request with PI that should be prepared 135 * 136 * For Type 1/Type 2, the virtual start sector is the one that was 137 * originally submitted by the block layer for the ref_tag usage. Due to 138 * partitioning, MD/DM cloning, etc. the actual physical start sector is 139 * likely to be different. Remap protection information to match the 140 * physical LBA. 141 */ 142static void t10_pi_type1_prepare(struct request *rq) 143{ 144 struct blk_integrity *bi = &rq->q->integrity; 145 const int tuple_sz = bi->tuple_size; 146 u32 ref_tag = t10_pi_ref_tag(rq); 147 u8 offset = bi->pi_offset; 148 struct bio *bio; 149 150 __rq_for_each_bio(bio, rq) { 151 struct bio_integrity_payload *bip = bio_integrity(bio); 152 u32 virt = bip_get_seed(bip) & 0xffffffff; 153 struct bio_vec iv; 154 struct bvec_iter iter; 155 156 /* Already remapped? */ 157 if (bip->bip_flags & BIP_MAPPED_INTEGRITY) 158 break; 159 160 bip_for_each_vec(iv, bip, iter) { 161 unsigned int j; 162 void *p; 163 164 p = bvec_kmap_local(&iv); 165 for (j = 0; j < iv.bv_len; j += tuple_sz) { 166 struct t10_pi_tuple *pi = p + offset; 167 168 if (be32_to_cpu(pi->ref_tag) == virt) 169 pi->ref_tag = cpu_to_be32(ref_tag); 170 virt++; 171 ref_tag++; 172 p += tuple_sz; 173 } 174 kunmap_local(p); 175 } 176 177 bip->bip_flags |= BIP_MAPPED_INTEGRITY; 178 } 179} 180 181/** 182 * t10_pi_type1_complete - prepare PI prior returning request to the blk layer 183 * @rq: request with PI that should be prepared 184 * @nr_bytes: total bytes to prepare 185 * 186 * For Type 1/Type 2, the virtual start sector is the one that was 187 * originally submitted by the block layer for the ref_tag usage. Due to 188 * partitioning, MD/DM cloning, etc. the actual physical start sector is 189 * likely to be different. Since the physical start sector was submitted 190 * to the device, we should remap it back to virtual values expected by the 191 * block layer. 192 */ 193static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes) 194{ 195 struct blk_integrity *bi = &rq->q->integrity; 196 unsigned intervals = nr_bytes >> bi->interval_exp; 197 const int tuple_sz = bi->tuple_size; 198 u32 ref_tag = t10_pi_ref_tag(rq); 199 u8 offset = bi->pi_offset; 200 struct bio *bio; 201 202 __rq_for_each_bio(bio, rq) { 203 struct bio_integrity_payload *bip = bio_integrity(bio); 204 u32 virt = bip_get_seed(bip) & 0xffffffff; 205 struct bio_vec iv; 206 struct bvec_iter iter; 207 208 bip_for_each_vec(iv, bip, iter) { 209 unsigned int j; 210 void *p; 211 212 p = bvec_kmap_local(&iv); 213 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) { 214 struct t10_pi_tuple *pi = p + offset; 215 216 if (be32_to_cpu(pi->ref_tag) == ref_tag) 217 pi->ref_tag = cpu_to_be32(virt); 218 virt++; 219 ref_tag++; 220 intervals--; 221 p += tuple_sz; 222 } 223 kunmap_local(p); 224 } 225 } 226} 227 228static blk_status_t t10_pi_type3_generate_crc(struct blk_integrity_iter *iter) 229{ 230 return t10_pi_generate(iter, t10_pi_crc_fn, T10_PI_TYPE3_PROTECTION); 231} 232 233static blk_status_t t10_pi_type3_generate_ip(struct blk_integrity_iter *iter) 234{ 235 return t10_pi_generate(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION); 236} 237 238static blk_status_t t10_pi_type3_verify_crc(struct blk_integrity_iter *iter) 239{ 240 return t10_pi_verify(iter, t10_pi_crc_fn, T10_PI_TYPE3_PROTECTION); 241} 242 243static blk_status_t t10_pi_type3_verify_ip(struct blk_integrity_iter *iter) 244{ 245 return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION); 246} 247 248/* Type 3 does not have a reference tag so no remapping is required. */ 249static void t10_pi_type3_prepare(struct request *rq) 250{ 251} 252 253/* Type 3 does not have a reference tag so no remapping is required. */ 254static void t10_pi_type3_complete(struct request *rq, unsigned int nr_bytes) 255{ 256} 257 258const struct blk_integrity_profile t10_pi_type1_crc = { 259 .name = "T10-DIF-TYPE1-CRC", 260 .generate_fn = t10_pi_type1_generate_crc, 261 .verify_fn = t10_pi_type1_verify_crc, 262 .prepare_fn = t10_pi_type1_prepare, 263 .complete_fn = t10_pi_type1_complete, 264}; 265EXPORT_SYMBOL(t10_pi_type1_crc); 266 267const struct blk_integrity_profile t10_pi_type1_ip = { 268 .name = "T10-DIF-TYPE1-IP", 269 .generate_fn = t10_pi_type1_generate_ip, 270 .verify_fn = t10_pi_type1_verify_ip, 271 .prepare_fn = t10_pi_type1_prepare, 272 .complete_fn = t10_pi_type1_complete, 273}; 274EXPORT_SYMBOL(t10_pi_type1_ip); 275 276const struct blk_integrity_profile t10_pi_type3_crc = { 277 .name = "T10-DIF-TYPE3-CRC", 278 .generate_fn = t10_pi_type3_generate_crc, 279 .verify_fn = t10_pi_type3_verify_crc, 280 .prepare_fn = t10_pi_type3_prepare, 281 .complete_fn = t10_pi_type3_complete, 282}; 283EXPORT_SYMBOL(t10_pi_type3_crc); 284 285const struct blk_integrity_profile t10_pi_type3_ip = { 286 .name = "T10-DIF-TYPE3-IP", 287 .generate_fn = t10_pi_type3_generate_ip, 288 .verify_fn = t10_pi_type3_verify_ip, 289 .prepare_fn = t10_pi_type3_prepare, 290 .complete_fn = t10_pi_type3_complete, 291}; 292EXPORT_SYMBOL(t10_pi_type3_ip); 293 294static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len) 295{ 296 return cpu_to_be64(crc64_rocksoft_update(crc, data, len)); 297} 298 299static blk_status_t ext_pi_crc64_generate(struct blk_integrity_iter *iter, 300 enum t10_dif_type type) 301{ 302 u8 offset = iter->pi_offset; 303 unsigned int i; 304 305 for (i = 0 ; i < iter->data_size ; i += iter->interval) { 306 struct crc64_pi_tuple *pi = iter->prot_buf + offset; 307 308 pi->guard_tag = ext_pi_crc64(0, iter->data_buf, iter->interval); 309 if (offset) 310 pi->guard_tag = ext_pi_crc64(be64_to_cpu(pi->guard_tag), 311 iter->prot_buf, offset); 312 pi->app_tag = 0; 313 314 if (type == T10_PI_TYPE1_PROTECTION) 315 put_unaligned_be48(iter->seed, pi->ref_tag); 316 else 317 put_unaligned_be48(0ULL, pi->ref_tag); 318 319 iter->data_buf += iter->interval; 320 iter->prot_buf += iter->tuple_size; 321 iter->seed++; 322 } 323 324 return BLK_STS_OK; 325} 326 327static bool ext_pi_ref_escape(u8 *ref_tag) 328{ 329 static u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; 330 331 return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0; 332} 333 334static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter, 335 enum t10_dif_type type) 336{ 337 u8 offset = iter->pi_offset; 338 unsigned int i; 339 340 for (i = 0; i < iter->data_size; i += iter->interval) { 341 struct crc64_pi_tuple *pi = iter->prot_buf + offset; 342 u64 ref, seed; 343 __be64 csum; 344 345 if (type == T10_PI_TYPE1_PROTECTION) { 346 if (pi->app_tag == T10_PI_APP_ESCAPE) 347 goto next; 348 349 ref = get_unaligned_be48(pi->ref_tag); 350 seed = lower_48_bits(iter->seed); 351 if (ref != seed) { 352 pr_err("%s: ref tag error at location %llu (rcvd %llu)\n", 353 iter->disk_name, seed, ref); 354 return BLK_STS_PROTECTION; 355 } 356 } else if (type == T10_PI_TYPE3_PROTECTION) { 357 if (pi->app_tag == T10_PI_APP_ESCAPE && 358 ext_pi_ref_escape(pi->ref_tag)) 359 goto next; 360 } 361 362 csum = ext_pi_crc64(0, iter->data_buf, iter->interval); 363 if (offset) 364 csum = ext_pi_crc64(be64_to_cpu(csum), iter->prot_buf, 365 offset); 366 367 if (pi->guard_tag != csum) { 368 pr_err("%s: guard tag error at sector %llu " \ 369 "(rcvd %016llx, want %016llx)\n", 370 iter->disk_name, (unsigned long long)iter->seed, 371 be64_to_cpu(pi->guard_tag), be64_to_cpu(csum)); 372 return BLK_STS_PROTECTION; 373 } 374 375next: 376 iter->data_buf += iter->interval; 377 iter->prot_buf += iter->tuple_size; 378 iter->seed++; 379 } 380 381 return BLK_STS_OK; 382} 383 384static blk_status_t ext_pi_type1_verify_crc64(struct blk_integrity_iter *iter) 385{ 386 return ext_pi_crc64_verify(iter, T10_PI_TYPE1_PROTECTION); 387} 388 389static blk_status_t ext_pi_type1_generate_crc64(struct blk_integrity_iter *iter) 390{ 391 return ext_pi_crc64_generate(iter, T10_PI_TYPE1_PROTECTION); 392} 393 394static void ext_pi_type1_prepare(struct request *rq) 395{ 396 struct blk_integrity *bi = &rq->q->integrity; 397 const int tuple_sz = bi->tuple_size; 398 u64 ref_tag = ext_pi_ref_tag(rq); 399 u8 offset = bi->pi_offset; 400 struct bio *bio; 401 402 __rq_for_each_bio(bio, rq) { 403 struct bio_integrity_payload *bip = bio_integrity(bio); 404 u64 virt = lower_48_bits(bip_get_seed(bip)); 405 struct bio_vec iv; 406 struct bvec_iter iter; 407 408 /* Already remapped? */ 409 if (bip->bip_flags & BIP_MAPPED_INTEGRITY) 410 break; 411 412 bip_for_each_vec(iv, bip, iter) { 413 unsigned int j; 414 void *p; 415 416 p = bvec_kmap_local(&iv); 417 for (j = 0; j < iv.bv_len; j += tuple_sz) { 418 struct crc64_pi_tuple *pi = p + offset; 419 u64 ref = get_unaligned_be48(pi->ref_tag); 420 421 if (ref == virt) 422 put_unaligned_be48(ref_tag, pi->ref_tag); 423 virt++; 424 ref_tag++; 425 p += tuple_sz; 426 } 427 kunmap_local(p); 428 } 429 430 bip->bip_flags |= BIP_MAPPED_INTEGRITY; 431 } 432} 433 434static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes) 435{ 436 struct blk_integrity *bi = &rq->q->integrity; 437 unsigned intervals = nr_bytes >> bi->interval_exp; 438 const int tuple_sz = bi->tuple_size; 439 u64 ref_tag = ext_pi_ref_tag(rq); 440 u8 offset = bi->pi_offset; 441 struct bio *bio; 442 443 __rq_for_each_bio(bio, rq) { 444 struct bio_integrity_payload *bip = bio_integrity(bio); 445 u64 virt = lower_48_bits(bip_get_seed(bip)); 446 struct bio_vec iv; 447 struct bvec_iter iter; 448 449 bip_for_each_vec(iv, bip, iter) { 450 unsigned int j; 451 void *p; 452 453 p = bvec_kmap_local(&iv); 454 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) { 455 struct crc64_pi_tuple *pi = p + offset; 456 u64 ref = get_unaligned_be48(pi->ref_tag); 457 458 if (ref == ref_tag) 459 put_unaligned_be48(virt, pi->ref_tag); 460 virt++; 461 ref_tag++; 462 intervals--; 463 p += tuple_sz; 464 } 465 kunmap_local(p); 466 } 467 } 468} 469 470static blk_status_t ext_pi_type3_verify_crc64(struct blk_integrity_iter *iter) 471{ 472 return ext_pi_crc64_verify(iter, T10_PI_TYPE3_PROTECTION); 473} 474 475static blk_status_t ext_pi_type3_generate_crc64(struct blk_integrity_iter *iter) 476{ 477 return ext_pi_crc64_generate(iter, T10_PI_TYPE3_PROTECTION); 478} 479 480const struct blk_integrity_profile ext_pi_type1_crc64 = { 481 .name = "EXT-DIF-TYPE1-CRC64", 482 .generate_fn = ext_pi_type1_generate_crc64, 483 .verify_fn = ext_pi_type1_verify_crc64, 484 .prepare_fn = ext_pi_type1_prepare, 485 .complete_fn = ext_pi_type1_complete, 486}; 487EXPORT_SYMBOL_GPL(ext_pi_type1_crc64); 488 489const struct blk_integrity_profile ext_pi_type3_crc64 = { 490 .name = "EXT-DIF-TYPE3-CRC64", 491 .generate_fn = ext_pi_type3_generate_crc64, 492 .verify_fn = ext_pi_type3_verify_crc64, 493 .prepare_fn = t10_pi_type3_prepare, 494 .complete_fn = t10_pi_type3_complete, 495}; 496EXPORT_SYMBOL_GPL(ext_pi_type3_crc64); 497 498MODULE_LICENSE("GPL"); 499MODULE_LICENSE("GPL"); 500