BUFInstructions.td revision 360784
1//===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 10def MUBUFAddr64 : ComplexPattern<i64, 9, "SelectMUBUFAddr64">; 11def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 12 13def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 14def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 15 16def MUBUFOffset : ComplexPattern<i64, 8, "SelectMUBUFOffset">; 17def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 18def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 19 20def BUFAddrKind { 21 int Offset = 0; 22 int OffEn = 1; 23 int IdxEn = 2; 24 int BothEn = 3; 25 int Addr64 = 4; 26} 27 28class getAddrName<int addrKind> { 29 string ret = 30 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 31 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 32 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 33 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 34 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 35 ""))))); 36} 37 38class MUBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43class MUBUFLdsTable <bit is_lds, string Name> { 44 bit IsLds = is_lds; 45 string OpName = Name; 46} 47 48class MTBUFAddr64Table <bit is_addr64, string Name> { 49 bit IsAddr64 = is_addr64; 50 string OpName = Name; 51} 52 53//===----------------------------------------------------------------------===// 54// MTBUF classes 55//===----------------------------------------------------------------------===// 56 57class MTBUFGetBaseOpcode<string Op> { 58 string ret = !subst("FORMAT_XY", "FORMAT_X", 59 !subst("FORMAT_XYZ", "FORMAT_X", 60 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 61} 62 63class getMTBUFElements<string Op> { 64 int ret = 1; 65} 66 67 68class MTBUF_Pseudo <string opName, dag outs, dag ins, 69 string asmOps, list<dag> pattern=[]> : 70 InstSI<outs, ins, "", pattern>, 71 SIMCInstr<opName, SIEncodingFamily.NONE> { 72 73 let isPseudo = 1; 74 let isCodeGenOnly = 1; 75 let Size = 8; 76 let UseNamedOperandTable = 1; 77 78 string Mnemonic = opName; 79 string AsmOperands = asmOps; 80 81 Instruction Opcode = !cast<Instruction>(NAME); 82 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 83 84 let VM_CNT = 1; 85 let EXP_CNT = 1; 86 let MTBUF = 1; 87 let Uses = [EXEC]; 88 let hasSideEffects = 0; 89 let SchedRW = [WriteVMEM]; 90 91 let AsmMatchConverter = "cvtMtbuf"; 92 93 bits<1> offen = 0; 94 bits<1> idxen = 0; 95 bits<1> addr64 = 0; 96 bits<1> has_vdata = 1; 97 bits<1> has_vaddr = 1; 98 bits<1> has_glc = 1; 99 bits<1> has_dlc = 1; 100 bits<1> glc_value = 0; // the value for glc if no such operand 101 bits<1> dlc_value = 0; // the value for dlc if no such operand 102 bits<1> has_srsrc = 1; 103 bits<1> has_soffset = 1; 104 bits<1> has_offset = 1; 105 bits<1> has_slc = 1; 106 bits<1> has_tfe = 1; 107 bits<4> elements = 0; 108} 109 110class MTBUF_Real <MTBUF_Pseudo ps> : 111 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 112 113 let isPseudo = 0; 114 let isCodeGenOnly = 0; 115 116 // copy relevant pseudo op flags 117 let SubtargetPredicate = ps.SubtargetPredicate; 118 let AsmMatchConverter = ps.AsmMatchConverter; 119 let Constraints = ps.Constraints; 120 let DisableEncoding = ps.DisableEncoding; 121 let TSFlags = ps.TSFlags; 122 123 bits<12> offset; 124 bits<1> glc; 125 bits<1> dlc; 126 bits<7> format; 127 bits<8> vaddr; 128 bits<8> vdata; 129 bits<7> srsrc; 130 bits<1> slc; 131 bits<1> tfe; 132 bits<8> soffset; 133 134 bits<4> dfmt = format{3-0}; 135 bits<3> nfmt = format{6-4}; 136} 137 138class getMTBUFInsDA<list<RegisterClass> vdataList, 139 list<RegisterClass> vaddrList=[]> { 140 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 141 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 142 dag InsNoData = !if(!empty(vaddrList), 143 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 144 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz), 145 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 146 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz) 147 ); 148 dag InsData = !if(!empty(vaddrList), 149 (ins vdataClass:$vdata, SReg_128:$srsrc, 150 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 151 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz), 152 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 153 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 154 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz) 155 ); 156 dag ret = !if(!empty(vdataList), InsNoData, InsData); 157} 158 159class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 160 dag ret = 161 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 162 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 163 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 164 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 165 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 166 (ins)))))); 167} 168 169class getMTBUFAsmOps<int addrKind> { 170 string Pfx = 171 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset", 172 !if(!eq(addrKind, BUFAddrKind.OffEn), 173 "$vaddr, $srsrc, $format, $soffset offen", 174 !if(!eq(addrKind, BUFAddrKind.IdxEn), 175 "$vaddr, $srsrc, $format, $soffset idxen", 176 !if(!eq(addrKind, BUFAddrKind.BothEn), 177 "$vaddr, $srsrc, $format, $soffset idxen offen", 178 !if(!eq(addrKind, BUFAddrKind.Addr64), 179 "$vaddr, $srsrc, $format, $soffset addr64", 180 ""))))); 181 string ret = Pfx # "$offset"; 182} 183 184class MTBUF_SetupAddr<int addrKind> { 185 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 186 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 187 188 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 189 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 190 191 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 192 193 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 194} 195 196class MTBUF_Load_Pseudo <string opName, 197 int addrKind, 198 RegisterClass vdataClass, 199 int elems, 200 list<dag> pattern=[], 201 // Workaround bug bz30254 202 int addrKindCopy = addrKind> 203 : MTBUF_Pseudo<opName, 204 (outs vdataClass:$vdata), 205 getMTBUFIns<addrKindCopy>.ret, 206 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 207 pattern>, 208 MTBUF_SetupAddr<addrKindCopy> { 209 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 210 let mayLoad = 1; 211 let mayStore = 0; 212 let elements = elems; 213} 214 215multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 216 int elems, ValueType load_vt = i32, 217 SDPatternOperator ld = null_frag> { 218 219 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 220 [(set load_vt:$vdata, 221 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format, 222 i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 223 MTBUFAddr64Table<0, NAME>; 224 225 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 226 [(set load_vt:$vdata, 227 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 228 i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 229 MTBUFAddr64Table<1, NAME>; 230 231 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 232 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 233 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 234 235 let DisableWQM = 1 in { 236 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 237 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 238 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 239 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 240 } 241} 242 243class MTBUF_Store_Pseudo <string opName, 244 int addrKind, 245 RegisterClass vdataClass, 246 int elems, 247 list<dag> pattern=[], 248 // Workaround bug bz30254 249 int addrKindCopy = addrKind, 250 RegisterClass vdataClassCopy = vdataClass> 251 : MTBUF_Pseudo<opName, 252 (outs), 253 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 254 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 255 pattern>, 256 MTBUF_SetupAddr<addrKindCopy> { 257 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 258 let mayLoad = 0; 259 let mayStore = 1; 260 let elements = elems; 261} 262 263multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 264 int elems, ValueType store_vt = i32, 265 SDPatternOperator st = null_frag> { 266 267 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 268 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 269 i16:$offset, i8:$format, i1:$glc, 270 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 271 MTBUFAddr64Table<0, NAME>; 272 273 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 274 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 275 i16:$offset, i8:$format, i1:$glc, 276 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 277 MTBUFAddr64Table<1, NAME>; 278 279 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 280 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 281 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 282 283 let DisableWQM = 1 in { 284 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 285 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 286 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 287 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 288 } 289} 290 291 292//===----------------------------------------------------------------------===// 293// MUBUF classes 294//===----------------------------------------------------------------------===// 295 296class MUBUFGetBaseOpcode<string Op> { 297 string ret = !subst("DWORDX2", "DWORD", 298 !subst("DWORDX3", "DWORD", 299 !subst("DWORDX4", "DWORD", Op))); 300} 301 302class MUBUF_Pseudo <string opName, dag outs, dag ins, 303 string asmOps, list<dag> pattern=[]> : 304 InstSI<outs, ins, "", pattern>, 305 SIMCInstr<opName, SIEncodingFamily.NONE> { 306 307 let isPseudo = 1; 308 let isCodeGenOnly = 1; 309 let Size = 8; 310 let UseNamedOperandTable = 1; 311 312 string Mnemonic = opName; 313 string AsmOperands = asmOps; 314 315 Instruction Opcode = !cast<Instruction>(NAME); 316 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 317 318 let VM_CNT = 1; 319 let EXP_CNT = 1; 320 let MUBUF = 1; 321 let Uses = [EXEC]; 322 let hasSideEffects = 0; 323 let SchedRW = [WriteVMEM]; 324 325 let AsmMatchConverter = "cvtMubuf"; 326 327 bits<1> offen = 0; 328 bits<1> idxen = 0; 329 bits<1> addr64 = 0; 330 bits<1> lds = 0; 331 bits<1> has_vdata = 1; 332 bits<1> has_vaddr = 1; 333 bits<1> has_glc = 1; 334 bits<1> has_dlc = 1; 335 bits<1> glc_value = 0; // the value for glc if no such operand 336 bits<1> dlc_value = 0; // the value for dlc if no such operand 337 bits<1> has_srsrc = 1; 338 bits<1> has_soffset = 1; 339 bits<1> has_offset = 1; 340 bits<1> has_slc = 1; 341 bits<1> has_tfe = 1; 342 bits<4> elements = 0; 343} 344 345class MUBUF_Real <MUBUF_Pseudo ps> : 346 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 347 348 let isPseudo = 0; 349 let isCodeGenOnly = 0; 350 351 // copy relevant pseudo op flags 352 let SubtargetPredicate = ps.SubtargetPredicate; 353 let AsmMatchConverter = ps.AsmMatchConverter; 354 let Constraints = ps.Constraints; 355 let DisableEncoding = ps.DisableEncoding; 356 let TSFlags = ps.TSFlags; 357 358 bits<12> offset; 359 bits<1> glc; 360 bits<1> dlc; 361 bits<8> vaddr; 362 bits<8> vdata; 363 bits<7> srsrc; 364 bits<1> slc; 365 bits<1> tfe; 366 bits<8> soffset; 367} 368 369 370// For cache invalidation instructions. 371class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 372 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 373 374 let AsmMatchConverter = ""; 375 376 let hasSideEffects = 1; 377 let mayStore = 1; 378 379 // Set everything to 0. 380 let offen = 0; 381 let idxen = 0; 382 let addr64 = 0; 383 let has_vdata = 0; 384 let has_vaddr = 0; 385 let has_glc = 0; 386 let has_dlc = 0; 387 let glc_value = 0; 388 let dlc_value = 0; 389 let has_srsrc = 0; 390 let has_soffset = 0; 391 let has_offset = 0; 392 let has_slc = 0; 393 let has_tfe = 0; 394} 395 396class getMUBUFInsDA<list<RegisterClass> vdataList, 397 list<RegisterClass> vaddrList=[], 398 bit isLds = 0> { 399 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 400 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 401 dag InsNoData = !if(!empty(vaddrList), 402 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 403 offset:$offset, GLC:$glc, SLC:$slc), 404 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 405 offset:$offset, GLC:$glc, SLC:$slc) 406 ); 407 dag InsData = !if(!empty(vaddrList), 408 (ins vdataClass:$vdata, SReg_128:$srsrc, 409 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 410 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 411 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 412 ); 413 dag ret = !con( 414 !if(!empty(vdataList), InsNoData, InsData), 415 !if(isLds, (ins DLC:$dlc, SWZ:$swz), (ins TFE:$tfe, DLC:$dlc,SWZ:$swz)) 416 ); 417} 418 419class getMUBUFElements<ValueType vt> { 420 // eq does not support ValueType for some reason. 421 string vtAsStr = !cast<string>(vt); 422 423 int ret = 424 !if(!eq(vtAsStr, "f16"), 1, 425 !if(!eq(vtAsStr, "v2f16"), 2, 426 !if(!eq(vtAsStr, "v3f16"), 3, 427 !if(!eq(vtAsStr, "v4f16"), 4, 428 !if(!eq(vt.Size, 32), 1, 429 !if(!eq(vt.Size, 64), 2, 430 !if(!eq(vt.Size, 96), 3, 431 !if(!eq(vt.Size, 128), 4, 0) 432 ) 433 ) 434 ) 435 ) 436 ) 437 ) 438 ); 439} 440 441class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 442 dag ret = 443 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 444 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 445 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 446 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 447 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 448 (ins)))))); 449} 450 451class getMUBUFAsmOps<int addrKind> { 452 string Pfx = 453 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 454 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 455 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 456 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 457 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 458 ""))))); 459 string ret = Pfx # "$offset"; 460} 461 462class MUBUF_SetupAddr<int addrKind> { 463 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 464 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 465 466 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 467 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 468 469 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 470 471 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 472} 473 474class MUBUF_Load_Pseudo <string opName, 475 int addrKind, 476 ValueType vdata_vt, 477 bit HasTiedDest = 0, 478 bit isLds = 0, 479 list<dag> pattern=[], 480 // Workaround bug bz30254 481 int addrKindCopy = addrKind> 482 : MUBUF_Pseudo<opName, 483 (outs getVregSrcForVT<vdata_vt>.ret:$vdata), 484 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 485 !if(HasTiedDest, (ins getVregSrcForVT<vdata_vt>.ret:$vdata_in), (ins))), 486 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 487 !if(isLds, " lds", "$tfe") # "$dlc" # "$swz", 488 pattern>, 489 MUBUF_SetupAddr<addrKindCopy> { 490 let PseudoInstr = opName # !if(isLds, "_lds", "") # 491 "_" # getAddrName<addrKindCopy>.ret; 492 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 493 494 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 495 let mayLoad = 1; 496 let mayStore = 0; 497 let maybeAtomic = 1; 498 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 499 let has_tfe = !if(isLds, 0, 1); 500 let lds = isLds; 501 let elements = getMUBUFElements<vdata_vt>.ret; 502} 503 504class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 505 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 506 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 507>; 508 509class MUBUF_Addr64_Load_Pat <Instruction inst, 510 ValueType load_vt = i32, 511 SDPatternOperator ld = null_frag> : Pat < 512 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 513 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 514>; 515 516multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 517 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 518 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 519} 520 521 522// FIXME: tfe can't be an operand because it requires a separate 523// opcode because it needs an N+1 register class dest register. 524multiclass MUBUF_Pseudo_Loads<string opName, 525 ValueType load_vt = i32, 526 SDPatternOperator ld = null_frag, 527 bit TiedDest = 0, 528 bit isLds = 0> { 529 530 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>, 531 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 532 533 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, load_vt, TiedDest, isLds>, 534 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 535 536 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>; 537 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>; 538 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>; 539 540 let DisableWQM = 1 in { 541 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>; 542 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>; 543 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>; 544 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>; 545 } 546} 547 548multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32, 549 SDPatternOperator ld_nolds = null_frag, 550 SDPatternOperator ld_lds = null_frag> { 551 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>; 552 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>; 553} 554 555class MUBUF_Store_Pseudo <string opName, 556 int addrKind, 557 ValueType store_vt, 558 list<dag> pattern=[], 559 // Workaround bug bz30254 560 int addrKindCopy = addrKind> 561 : MUBUF_Pseudo<opName, 562 (outs), 563 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 564 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 565 pattern>, 566 MUBUF_SetupAddr<addrKindCopy> { 567 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 568 let mayLoad = 0; 569 let mayStore = 1; 570 let maybeAtomic = 1; 571 let elements = getMUBUFElements<store_vt>.ret; 572} 573 574multiclass MUBUF_Pseudo_Stores<string opName, 575 ValueType store_vt = i32, 576 SDPatternOperator st = null_frag> { 577 578 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt, 579 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 580 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 581 MUBUFAddr64Table<0, NAME>; 582 583 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt, 584 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 585 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 586 MUBUFAddr64Table<1, NAME>; 587 588 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>; 589 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>; 590 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>; 591 592 let DisableWQM = 1 in { 593 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt>; 594 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>; 595 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>; 596 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>; 597 } 598} 599 600class MUBUF_Pseudo_Store_Lds<string opName> 601 : MUBUF_Pseudo<opName, 602 (outs), 603 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc, SWZ:$swz), 604 " $srsrc, $soffset$offset lds$glc$slc$swz"> { 605 let mayLoad = 0; 606 let mayStore = 1; 607 let maybeAtomic = 1; 608 609 let has_vdata = 0; 610 let has_vaddr = 0; 611 let has_tfe = 0; 612 let lds = 1; 613 614 let Uses = [EXEC, M0]; 615 let AsmMatchConverter = "cvtMubufLds"; 616} 617 618class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 619 list<RegisterClass> vaddrList=[]> { 620 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 621 dag ret = !if(vdata_in, 622 !if(!empty(vaddrList), 623 (ins vdataClass:$vdata_in, 624 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 625 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 626 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 627 ), 628 !if(!empty(vaddrList), 629 (ins vdataClass:$vdata, 630 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 631 (ins vdataClass:$vdata, vaddrClass:$vaddr, 632 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 633 )); 634} 635 636class getMUBUFAtomicIns<int addrKind, 637 RegisterClass vdataClass, 638 bit vdata_in, 639 // Workaround bug bz30254 640 RegisterClass vdataClassCopy=vdataClass> { 641 dag ret = 642 !if(!eq(addrKind, BUFAddrKind.Offset), 643 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 644 !if(!eq(addrKind, BUFAddrKind.OffEn), 645 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 646 !if(!eq(addrKind, BUFAddrKind.IdxEn), 647 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 648 !if(!eq(addrKind, BUFAddrKind.BothEn), 649 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 650 !if(!eq(addrKind, BUFAddrKind.Addr64), 651 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 652 (ins)))))); 653} 654 655class MUBUF_Atomic_Pseudo<string opName, 656 int addrKind, 657 dag outs, 658 dag ins, 659 string asmOps, 660 list<dag> pattern=[], 661 // Workaround bug bz30254 662 int addrKindCopy = addrKind> 663 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 664 MUBUF_SetupAddr<addrKindCopy> { 665 let mayStore = 1; 666 let mayLoad = 1; 667 let hasPostISelHook = 1; 668 let hasSideEffects = 1; 669 let DisableWQM = 1; 670 let has_glc = 0; 671 let has_dlc = 0; 672 let has_tfe = 0; 673 let maybeAtomic = 1; 674} 675 676class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 677 RegisterClass vdataClass, 678 list<dag> pattern=[], 679 // Workaround bug bz30254 680 int addrKindCopy = addrKind, 681 RegisterClass vdataClassCopy = vdataClass> 682 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 683 (outs), 684 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 685 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 686 pattern>, 687 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 688 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 689 let glc_value = 0; 690 let dlc_value = 0; 691 let AsmMatchConverter = "cvtMubufAtomic"; 692} 693 694class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 695 RegisterClass vdataClass, 696 list<dag> pattern=[], 697 // Workaround bug bz30254 698 int addrKindCopy = addrKind, 699 RegisterClass vdataClassCopy = vdataClass> 700 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 701 (outs vdataClassCopy:$vdata), 702 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 703 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 704 pattern>, 705 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 706 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 707 let glc_value = 1; 708 let dlc_value = 0; 709 let Constraints = "$vdata = $vdata_in"; 710 let DisableEncoding = "$vdata_in"; 711 let AsmMatchConverter = "cvtMubufAtomicReturn"; 712} 713 714multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 715 RegisterClass vdataClass, 716 ValueType vdataType, 717 SDPatternOperator atomic, 718 bit isFP = isFloatType<vdataType>.ret> { 719 let FPAtomic = isFP in 720 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 721 MUBUFAddr64Table <0, NAME>; 722 723 let FPAtomic = isFP in 724 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 725 MUBUFAddr64Table <1, NAME>; 726 727 let FPAtomic = isFP in 728 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 729 730 let FPAtomic = isFP in 731 732 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 733 734 let FPAtomic = isFP in 735 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 736} 737 738multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 739 RegisterClass vdataClass, 740 ValueType vdataType, 741 SDPatternOperator atomic, 742 bit isFP = isFloatType<vdataType>.ret> { 743 let FPAtomic = isFP in 744 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 745 [(set vdataType:$vdata, 746 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 747 vdataType:$vdata_in))]>, 748 MUBUFAddr64Table <0, NAME # "_RTN">; 749 750 let FPAtomic = isFP in 751 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 752 [(set vdataType:$vdata, 753 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 754 vdataType:$vdata_in))]>, 755 MUBUFAddr64Table <1, NAME # "_RTN">; 756 757 let FPAtomic = isFP in 758 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 759 760 let FPAtomic = isFP in 761 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 762 763 let FPAtomic = isFP in 764 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 765} 766 767multiclass MUBUF_Pseudo_Atomics <string opName, 768 RegisterClass vdataClass, 769 ValueType vdataType, 770 SDPatternOperator atomic> : 771 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 772 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 773 774 775//===----------------------------------------------------------------------===// 776// MUBUF Instructions 777//===----------------------------------------------------------------------===// 778 779defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 780 "buffer_load_format_x", f32 781>; 782defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 783 "buffer_load_format_xy", v2f32 784>; 785defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 786 "buffer_load_format_xyz", v3f32 787>; 788defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 789 "buffer_load_format_xyzw", v4f32 790>; 791defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 792 "buffer_store_format_x", f32 793>; 794defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 795 "buffer_store_format_xy", v2f32 796>; 797defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 798 "buffer_store_format_xyz", v3f32 799>; 800defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 801 "buffer_store_format_xyzw", v4f32 802>; 803 804let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 805 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 806 "buffer_load_format_d16_x", i32 807 >; 808 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 809 "buffer_load_format_d16_xy", v2i32 810 >; 811 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 812 "buffer_load_format_d16_xyz", v3i32 813 >; 814 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 815 "buffer_load_format_d16_xyzw", v4i32 816 >; 817 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 818 "buffer_store_format_d16_x", i32 819 >; 820 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 821 "buffer_store_format_d16_xy", v2i32 822 >; 823 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 824 "buffer_store_format_d16_xyz", v3i32 825 >; 826 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 827 "buffer_store_format_d16_xyzw", v4i32 828 >; 829} // End HasUnpackedD16VMem. 830 831let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 832 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 833 "buffer_load_format_d16_x", f16 834 >; 835 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 836 "buffer_load_format_d16_xy", v2f16 837 >; 838 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 839 "buffer_load_format_d16_xyz", v3f16 840 >; 841 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 842 "buffer_load_format_d16_xyzw", v4f16 843 >; 844 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 845 "buffer_store_format_d16_x", f16 846 >; 847 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 848 "buffer_store_format_d16_xy", v2f16 849 >; 850 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 851 "buffer_store_format_d16_xyz", v3f16 852 >; 853 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 854 "buffer_store_format_d16_xyzw", v4f16 855 >; 856} // End HasPackedD16VMem. 857 858defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 859 "buffer_load_ubyte", i32 860>; 861defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 862 "buffer_load_sbyte", i32 863>; 864defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 865 "buffer_load_ushort", i32 866>; 867defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 868 "buffer_load_sshort", i32 869>; 870defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 871 "buffer_load_dword", i32 872>; 873defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 874 "buffer_load_dwordx2", v2i32 875>; 876defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 877 "buffer_load_dwordx3", v3i32 878>; 879defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 880 "buffer_load_dwordx4", v4i32 881>; 882 883defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 884defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 885defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 886defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 887defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 888defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 889defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 890defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 891defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 892defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 893 894// This is not described in AMD documentation, 895// but 'lds' versions of these opcodes are available 896// in at least GFX8+ chips. See Bug 37653. 897let SubtargetPredicate = isGFX8GFX9 in { 898defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 899 "buffer_load_dwordx2", v2i32, null_frag, 0, 1 900>; 901defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 902 "buffer_load_dwordx3", v3i32, null_frag, 0, 1 903>; 904defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 905 "buffer_load_dwordx4", v4i32, null_frag, 0, 1 906>; 907} 908 909defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 910 "buffer_store_byte", i32, truncstorei8_global 911>; 912defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 913 "buffer_store_short", i32, truncstorei16_global 914>; 915defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 916 "buffer_store_dword", i32, store_global 917>; 918defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 919 "buffer_store_dwordx2", v2i32, store_global 920>; 921defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 922 "buffer_store_dwordx3", v3i32, store_global 923>; 924defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 925 "buffer_store_dwordx4", v4i32, store_global 926>; 927defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 928 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32 929>; 930defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 931 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 932>; 933defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 934 "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32 935>; 936defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 937 "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32 938>; 939defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 940 "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32 941>; 942defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 943 "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32 944>; 945defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 946 "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32 947>; 948defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 949 "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32 950>; 951defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 952 "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32 953>; 954defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 955 "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32 956>; 957defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 958 "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32 959>; 960defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 961 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32 962>; 963defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 964 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32 965>; 966defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 967 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64 968>; 969defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 970 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 971>; 972defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 973 "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64 974>; 975defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 976 "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64 977>; 978defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 979 "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64 980>; 981defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 982 "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64 983>; 984defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 985 "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64 986>; 987defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 988 "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64 989>; 990defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 991 "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64 992>; 993defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 994 "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64 995>; 996defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 997 "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64 998>; 999defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1000 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64 1001>; 1002defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1003 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64 1004>; 1005 1006let SubtargetPredicate = isGFX8GFX9 in { 1007def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1008} 1009 1010let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1011/* 1012defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1013defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1014*/ 1015 1016def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1017 int_amdgcn_buffer_wbinvl1_sc>; 1018} 1019 1020let SubtargetPredicate = isGFX6GFX7GFX10 in { 1021 1022defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1023 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1024>; 1025defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1026 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1027>; 1028defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1029 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1030>; 1031defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1032 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1033>; 1034defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1035 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1036>; 1037defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1038 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1039>; 1040 1041} 1042 1043let SubtargetPredicate = HasD16LoadStore in { 1044 1045defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1046 "buffer_load_ubyte_d16", i32, null_frag, 1 1047>; 1048 1049defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1050 "buffer_load_ubyte_d16_hi", i32, null_frag, 1 1051>; 1052 1053defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1054 "buffer_load_sbyte_d16", i32, null_frag, 1 1055>; 1056 1057defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1058 "buffer_load_sbyte_d16_hi", i32, null_frag, 1 1059>; 1060 1061defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1062 "buffer_load_short_d16", i32, null_frag, 1 1063>; 1064 1065defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1066 "buffer_load_short_d16_hi", i32, null_frag, 1 1067>; 1068 1069defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1070 "buffer_store_byte_d16_hi", i32 1071>; 1072 1073defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1074 "buffer_store_short_d16_hi", i32 1075>; 1076 1077defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1078 "buffer_load_format_d16_hi_x", i32 1079>; 1080defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1081 "buffer_store_format_d16_hi_x", i32 1082>; 1083 1084} // End HasD16LoadStore 1085 1086def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1087 int_amdgcn_buffer_wbinvl1>; 1088 1089let SubtargetPredicate = HasAtomicFaddInsts in { 1090 1091defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1092 "buffer_atomic_add_f32", VGPR_32, f32, atomic_fadd_global_noret 1093>; 1094defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1095 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_pk_fadd_global_noret 1096>; 1097 1098} // End SubtargetPredicate = HasAtomicFaddInsts 1099 1100//===----------------------------------------------------------------------===// 1101// MTBUF Instructions 1102//===----------------------------------------------------------------------===// 1103 1104defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1105defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1106defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1107defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1108defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1109defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1110defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1111defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1112 1113let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1114 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1115 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1116 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1117 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1118 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1119 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1120 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1121 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1122} // End HasUnpackedD16VMem. 1123 1124let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1125 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1126 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1127 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1128 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1129 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1130 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1131 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1132 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1133} // End HasPackedD16VMem. 1134 1135let SubtargetPredicate = isGFX7Plus in { 1136 1137//===----------------------------------------------------------------------===// 1138// Instruction definitions for CI and newer. 1139//===----------------------------------------------------------------------===// 1140 1141def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1142 int_amdgcn_buffer_wbinvl1_vol>; 1143 1144} // End let SubtargetPredicate = isGFX7Plus 1145 1146let SubtargetPredicate = isGFX10Plus in { 1147 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1148 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1149} // End SubtargetPredicate = isGFX10Plus 1150 1151//===----------------------------------------------------------------------===// 1152// MUBUF Patterns 1153//===----------------------------------------------------------------------===// 1154 1155def extract_glc : SDNodeXForm<imm, [{ 1156 return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i8); 1157}]>; 1158 1159def extract_slc : SDNodeXForm<imm, [{ 1160 return CurDAG->getTargetConstant((N->getZExtValue() >> 1) & 1, SDLoc(N), MVT::i8); 1161}]>; 1162 1163def extract_dlc : SDNodeXForm<imm, [{ 1164 return CurDAG->getTargetConstant((N->getZExtValue() >> 2) & 1, SDLoc(N), MVT::i8); 1165}]>; 1166 1167def extract_swz : SDNodeXForm<imm, [{ 1168 return CurDAG->getTargetConstant((N->getZExtValue() >> 3) & 1, SDLoc(N), MVT::i8); 1169}]>; 1170 1171//===----------------------------------------------------------------------===// 1172// buffer_load/store_format patterns 1173//===----------------------------------------------------------------------===// 1174 1175multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1176 string opcode> { 1177 def : GCNPat< 1178 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1179 timm:$auxiliary, 0)), 1180 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1181 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1182 (extract_swz $auxiliary)) 1183 >; 1184 1185 def : GCNPat< 1186 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1187 timm:$auxiliary, 0)), 1188 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1189 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1190 (extract_swz $auxiliary)) 1191 >; 1192 1193 def : GCNPat< 1194 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1195 timm:$auxiliary, timm)), 1196 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1197 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1198 (extract_swz $auxiliary)) 1199 >; 1200 1201 def : GCNPat< 1202 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1203 timm:$auxiliary, timm)), 1204 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1205 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1206 $rsrc, $soffset, (as_i16imm $offset), 1207 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1208 (extract_swz $auxiliary)) 1209 >; 1210} 1211 1212defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1213defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1214defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1215defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1216defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1217defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1218defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1219defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1220 1221let SubtargetPredicate = HasUnpackedD16VMem in { 1222 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1223 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1224 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1225 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1226} // End HasUnpackedD16VMem. 1227 1228let SubtargetPredicate = HasPackedD16VMem in { 1229 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1230 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1231 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1232 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1233 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1234 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1235} // End HasPackedD16VMem. 1236 1237defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1238defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1239defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1240defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1241defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1242defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1243defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1244defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1245defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1246defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1247defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1248defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1249defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1250defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1251defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1252defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1253 1254multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1255 string opcode> { 1256 def : GCNPat< 1257 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1258 timm:$auxiliary, 0), 1259 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset), 1260 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1261 (extract_swz $auxiliary)) 1262 >; 1263 1264 def : GCNPat< 1265 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1266 timm:$auxiliary, 0), 1267 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1268 (as_i16imm $offset), (extract_glc $auxiliary), 1269 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1270 (extract_swz $auxiliary)) 1271 >; 1272 1273 def : GCNPat< 1274 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1275 timm:$auxiliary, timm), 1276 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1277 (as_i16imm $offset), (extract_glc $auxiliary), 1278 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1279 (extract_swz $auxiliary)) 1280 >; 1281 1282 def : GCNPat< 1283 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1284 timm:$auxiliary, timm), 1285 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1286 $vdata, 1287 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1288 $rsrc, $soffset, (as_i16imm $offset), (extract_glc $auxiliary), 1289 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1290 (extract_swz $auxiliary)) 1291 >; 1292} 1293 1294defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1295defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1296defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1297defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1298defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1299defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1300defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1301defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1302 1303let SubtargetPredicate = HasUnpackedD16VMem in { 1304 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1305 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1306 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1307 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1308} // End HasUnpackedD16VMem. 1309 1310let SubtargetPredicate = HasPackedD16VMem in { 1311 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1312 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1313 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1314 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1315 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1316 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1317} // End HasPackedD16VMem. 1318 1319defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1320defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1321defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1322defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1323defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1324defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1325defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1326defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1327defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1328defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1329defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1330defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1331defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1332defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1333 1334//===----------------------------------------------------------------------===// 1335// buffer_atomic patterns 1336//===----------------------------------------------------------------------===// 1337 1338multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt, 1339 string opcode> { 1340 def : GCNPat< 1341 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 1342 0, i32:$soffset, timm:$offset, 1343 timm:$cachepolicy, 0)), 1344 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset, 1345 (as_i16imm $offset), (extract_slc $cachepolicy)) 1346 >; 1347 1348 def : GCNPat< 1349 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1350 0, i32:$soffset, timm:$offset, 1351 timm:$cachepolicy, timm)), 1352 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset, 1353 (as_i16imm $offset), (extract_slc $cachepolicy)) 1354 >; 1355 1356 def : GCNPat< 1357 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 1358 i32:$voffset, i32:$soffset, timm:$offset, 1359 timm:$cachepolicy, 0)), 1360 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset, 1361 (as_i16imm $offset), (extract_slc $cachepolicy)) 1362 >; 1363 1364 def : GCNPat< 1365 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1366 i32:$voffset, i32:$soffset, timm:$offset, 1367 timm:$cachepolicy, timm)), 1368 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1369 $vdata_in, 1370 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1371 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)) 1372 >; 1373} 1374 1375defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">; 1376defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">; 1377defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">; 1378defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">; 1379defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">; 1380defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">; 1381defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">; 1382defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">; 1383defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">; 1384defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">; 1385defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">; 1386defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">; 1387defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">; 1388defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64, "BUFFER_ATOMIC_ADD_X2">; 1389defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">; 1390defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">; 1391defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">; 1392defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">; 1393defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">; 1394defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">; 1395defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">; 1396defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">; 1397defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">; 1398defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">; 1399 1400multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1401 string opcode> { 1402 def : GCNPat< 1403 (name vt:$vdata_in, v4i32:$rsrc, 0, 1404 0, i32:$soffset, timm:$offset, 1405 timm:$cachepolicy, 0), 1406 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset, 1407 (as_i16imm $offset), (extract_slc $cachepolicy)) 1408 >; 1409 1410 def : GCNPat< 1411 (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1412 0, i32:$soffset, timm:$offset, 1413 timm:$cachepolicy, timm), 1414 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset, 1415 (as_i16imm $offset), (extract_slc $cachepolicy)) 1416 >; 1417 1418 def : GCNPat< 1419 (name vt:$vdata_in, v4i32:$rsrc, 0, 1420 i32:$voffset, i32:$soffset, timm:$offset, 1421 timm:$cachepolicy, 0), 1422 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset, 1423 (as_i16imm $offset), (extract_slc $cachepolicy)) 1424 >; 1425 1426 def : GCNPat< 1427 (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1428 i32:$voffset, i32:$soffset, timm:$offset, 1429 timm:$cachepolicy, timm), 1430 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1431 $vdata_in, 1432 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1433 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)) 1434 >; 1435} 1436 1437defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1438defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1439 1440def : GCNPat< 1441 (SIbuffer_atomic_cmpswap 1442 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1443 0, i32:$soffset, timm:$offset, 1444 timm:$cachepolicy, 0), 1445 (EXTRACT_SUBREG 1446 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1447 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1448 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1449 sub0) 1450>; 1451 1452def : GCNPat< 1453 (SIbuffer_atomic_cmpswap 1454 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1455 0, i32:$soffset, timm:$offset, 1456 timm:$cachepolicy, timm), 1457 (EXTRACT_SUBREG 1458 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1459 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1460 $vindex, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1461 sub0) 1462>; 1463 1464def : GCNPat< 1465 (SIbuffer_atomic_cmpswap 1466 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1467 i32:$voffset, i32:$soffset, timm:$offset, 1468 timm:$cachepolicy, 0), 1469 (EXTRACT_SUBREG 1470 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1471 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1472 $voffset, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1473 sub0) 1474>; 1475 1476def : GCNPat< 1477 (SIbuffer_atomic_cmpswap 1478 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1479 i32:$voffset, i32:$soffset, timm:$offset, 1480 timm:$cachepolicy, timm), 1481 (EXTRACT_SUBREG 1482 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1483 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1484 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1485 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1486 sub0) 1487>; 1488 1489class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1490 PatFrag constant_ld> : GCNPat < 1491 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1492 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1493 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1494 >; 1495 1496multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1497 ValueType vt, PatFrag atomic_ld> { 1498 def : GCNPat < 1499 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1500 i16:$offset, i1:$slc))), 1501 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0) 1502 >; 1503 1504 def : GCNPat < 1505 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1506 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0) 1507 >; 1508} 1509 1510let SubtargetPredicate = isGFX6GFX7 in { 1511def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1512def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1513def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1514def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1515def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1516def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1517 1518defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1519defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1520} // End SubtargetPredicate = isGFX6GFX7 1521 1522multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1523 PatFrag ld> { 1524 1525 def : GCNPat < 1526 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1527 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1528 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1529 >; 1530} 1531 1532let OtherPredicates = [Has16BitInsts] in { 1533 1534defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1535defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1536defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1537defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1538defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1539defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1540 1541defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1542 1543} // End OtherPredicates = [Has16BitInsts] 1544 1545multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1546 MUBUF_Pseudo InstrOffset, 1547 ValueType vt, PatFrag ld> { 1548 def : GCNPat < 1549 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1550 i32:$soffset, u16imm:$offset))), 1551 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1552 >; 1553 1554 def : GCNPat < 1555 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1556 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1557 >; 1558} 1559 1560// XXX - Is it possible to have a complex pattern in a PatFrag? 1561multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1562 MUBUF_Pseudo InstrOffset, 1563 ValueType vt, PatFrag ld_frag> { 1564 def : GCNPat < 1565 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1566 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1567 >; 1568 1569 def : GCNPat < 1570 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1571 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1572 >; 1573} 1574 1575defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1576defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1577defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1578defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1579defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1580defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1581defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1582defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1583defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1584defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1585 1586foreach vt = Reg32Types.types in { 1587defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>; 1588} 1589defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1590defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1591defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1592 1593let OtherPredicates = [D16PreservesUnusedBits] in { 1594defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1595defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1596defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1597defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1598defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1599defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1600 1601defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1602defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1603defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1604defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1605defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1606defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1607} 1608 1609multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1610 ValueType vt, PatFrag atomic_st> { 1611 // Store follows atomic op convention so address is first 1612 def : GCNPat < 1613 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1614 i16:$offset, i1:$slc), vt:$val), 1615 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0) 1616 >; 1617 1618 def : GCNPat < 1619 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1620 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0) 1621 >; 1622} 1623let SubtargetPredicate = isGFX6GFX7 in { 1624defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>; 1625defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>; 1626} // End Predicates = isGFX6GFX7 1627 1628 1629multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1630 PatFrag st> { 1631 1632 def : GCNPat < 1633 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1634 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)), 1635 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1636 >; 1637} 1638 1639defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1640defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1641 1642multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1643 MUBUF_Pseudo InstrOffset, 1644 ValueType vt, PatFrag st, 1645 RegisterClass rc = VGPR_32> { 1646 def : GCNPat < 1647 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1648 i32:$soffset, u16imm:$offset)), 1649 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1650 >; 1651 1652 def : GCNPat < 1653 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1654 u16imm:$offset)), 1655 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1656 >; 1657} 1658 1659defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1660defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1661defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1662defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1663 1664foreach vt = Reg32Types.types in { 1665defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1666} 1667 1668defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1669defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1670defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1671 1672 1673let OtherPredicates = [D16PreservesUnusedBits] in { 1674 // Hiding the extract high pattern in the PatFrag seems to not 1675 // automatically increase the complexity. 1676let AddedComplexity = 1 in { 1677defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1678defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1679} 1680} 1681 1682//===----------------------------------------------------------------------===// 1683// MTBUF Patterns 1684//===----------------------------------------------------------------------===// 1685 1686//===----------------------------------------------------------------------===// 1687// tbuffer_load/store_format patterns 1688//===----------------------------------------------------------------------===// 1689 1690multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1691 string opcode> { 1692 def : GCNPat< 1693 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1694 timm:$format, timm:$auxiliary, 0)), 1695 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1696 (as_i8imm $format), 1697 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1698 (extract_swz $auxiliary)) 1699 >; 1700 1701 def : GCNPat< 1702 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1703 timm:$format, timm:$auxiliary, timm)), 1704 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1705 (as_i8imm $format), 1706 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1707 (extract_swz $auxiliary)) 1708 >; 1709 1710 def : GCNPat< 1711 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1712 timm:$format, timm:$auxiliary, 0)), 1713 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1714 (as_i8imm $format), 1715 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1716 (extract_swz $auxiliary)) 1717 >; 1718 1719 def : GCNPat< 1720 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1721 timm:$format, timm:$auxiliary, timm)), 1722 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1723 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1724 $rsrc, $soffset, (as_i16imm $offset), 1725 (as_i8imm $format), 1726 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1727 (extract_swz $auxiliary)) 1728 >; 1729} 1730 1731defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1732defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1733defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1734defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1735defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1736defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1737defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1738defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1739 1740let SubtargetPredicate = HasUnpackedD16VMem in { 1741 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1742 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1743 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1744} // End HasUnpackedD16VMem. 1745 1746let SubtargetPredicate = HasPackedD16VMem in { 1747 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1748 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1749 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1750} // End HasPackedD16VMem. 1751 1752multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1753 string opcode> { 1754 def : GCNPat< 1755 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1756 timm:$format, timm:$auxiliary, 0), 1757 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, 1758 (as_i16imm $offset), (as_i8imm $format), 1759 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1760 (extract_swz $auxiliary)) 1761 >; 1762 1763 def : GCNPat< 1764 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1765 timm:$format, timm:$auxiliary, timm), 1766 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1767 (as_i16imm $offset), (as_i8imm $format), 1768 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1769 (extract_swz $auxiliary)) 1770 >; 1771 1772 def : GCNPat< 1773 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1774 timm:$format, timm:$auxiliary, 0), 1775 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1776 (as_i16imm $offset), (as_i8imm $format), 1777 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1778 (extract_swz $auxiliary)) 1779 >; 1780 1781 def : GCNPat< 1782 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1783 timm:$offset, timm:$format, timm:$auxiliary, timm), 1784 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1785 $vdata, 1786 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1787 $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format), 1788 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1789 (extract_swz $auxiliary)) 1790 >; 1791} 1792 1793defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1794defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1795defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1796defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1797defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1798defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1799defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1800defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1801 1802let SubtargetPredicate = HasUnpackedD16VMem in { 1803 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1804 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1805 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1806} // End HasUnpackedD16VMem. 1807 1808let SubtargetPredicate = HasPackedD16VMem in { 1809 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1810 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1811 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1812} // End HasPackedD16VMem. 1813 1814//===----------------------------------------------------------------------===// 1815// Target-specific instruction encodings. 1816//===----------------------------------------------------------------------===// 1817 1818//===----------------------------------------------------------------------===// 1819// Base ENC_MUBUF for GFX6, GFX7, GFX10. 1820//===----------------------------------------------------------------------===// 1821 1822class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 1823 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 1824 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1825 let Inst{12} = ps.offen; 1826 let Inst{13} = ps.idxen; 1827 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1828 let Inst{16} = !if(ps.lds, 1, 0); 1829 let Inst{24-18} = op; 1830 let Inst{31-26} = 0x38; 1831 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1832 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1833 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1834 let Inst{54} = !if(ps.has_slc, slc, ?); 1835 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1836 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1837} 1838 1839class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 1840 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 1841 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 1842 let Inst{25} = op{7}; 1843} 1844 1845class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 1846 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 1847 let Inst{15} = ps.addr64; 1848} 1849 1850//===----------------------------------------------------------------------===// 1851// MUBUF - GFX10. 1852//===----------------------------------------------------------------------===// 1853 1854let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 1855 multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName, 1856 string asmName> { 1857 def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> { 1858 MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName); 1859 let AsmString = asmName # ps.AsmOperands; 1860 } 1861 } 1862 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 1863 def _BOTHEN_gfx10 : 1864 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1865 def _IDXEN_gfx10 : 1866 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1867 def _OFFEN_gfx10 : 1868 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1869 def _OFFSET_gfx10 : 1870 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1871 } 1872 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 1873 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1874 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 1875 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1876 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 1877 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1878 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 1879 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1880 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 1881 1882 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1883 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 1884 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1885 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 1886 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1887 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 1888 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1889 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 1890 } 1891 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 1892 MUBUF_Real_AllAddr_gfx10<op> { 1893 def _BOTHEN_RTN_gfx10 : 1894 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1895 def _IDXEN_RTN_gfx10 : 1896 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1897 def _OFFEN_RTN_gfx10 : 1898 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1899 def _OFFSET_RTN_gfx10 : 1900 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1901 } 1902} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 1903 1904defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 1905defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 1906defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 1907defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 1908defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 1909defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 1910defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 1911defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 1912// FIXME-GFX10: Add following instructions: 1913//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 1914//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 1915defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 1916defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 1917defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 1918defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 1919defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 1920defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 1921defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 1922defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 1923 1924def BUFFER_GL0_INV_gfx10 : 1925 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 1926def BUFFER_GL1_INV_gfx10 : 1927 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 1928 1929//===----------------------------------------------------------------------===// 1930// MUBUF - GFX6, GFX7, GFX10. 1931//===----------------------------------------------------------------------===// 1932 1933let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 1934 multiclass MUBUF_Real_gfx6<bits<8> op> { 1935 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1936 } 1937} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 1938 1939let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 1940 multiclass MUBUF_Real_gfx7<bits<8> op> { 1941 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1942 } 1943} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 1944 1945let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 1946 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 1947 def _ADDR64_gfx6_gfx7 : 1948 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1949 def _BOTHEN_gfx6_gfx7 : 1950 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1951 def _IDXEN_gfx6_gfx7 : 1952 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1953 def _OFFEN_gfx6_gfx7 : 1954 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1955 def _OFFSET_gfx6_gfx7 : 1956 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1957 } 1958 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 1959 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1960 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 1961 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 1962 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 1963 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1964 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 1965 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1966 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 1967 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1968 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 1969 1970 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1971 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 1972 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 1973 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 1974 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1975 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 1976 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1977 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 1978 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1979 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 1980 } 1981 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> : 1982 MUBUF_Real_AllAddr_gfx6_gfx7<op> { 1983 def _ADDR64_RTN_gfx6_gfx7 : 1984 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1985 def _BOTHEN_RTN_gfx6_gfx7 : 1986 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1987 def _IDXEN_RTN_gfx6_gfx7 : 1988 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1989 def _OFFEN_RTN_gfx6_gfx7 : 1990 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1991 def _OFFSET_RTN_gfx6_gfx7 : 1992 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1993 } 1994} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 1995 1996multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 1997 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 1998 1999multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2000 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2001 2002multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2003 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2004 2005// FIXME-GFX6: Following instructions are available only on GFX6. 2006//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2007//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2008 2009defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2010defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2011defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2012defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2013defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2014defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2015defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2016defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2017defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2018defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2019defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2020defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2021defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2022defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2023defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2024defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2025defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2026defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2027defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2028defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2029defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2030defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2031 2032defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2033defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2034defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2035defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2036defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2037defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2038defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2039defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2040defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2041defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2042defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2043defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2044defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2045defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2046defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2047defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2048defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2049defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2050defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2051defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2052defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2053defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2054defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2055defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2056defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2057defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2058defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2059defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2060defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2061// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2062defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2063defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2064defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2065 2066defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2067defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2068def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2069 2070//===----------------------------------------------------------------------===// 2071// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2072//===----------------------------------------------------------------------===// 2073 2074class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2075 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2076 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2077 let Inst{12} = ps.offen; 2078 let Inst{13} = ps.idxen; 2079 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2080 let Inst{18-16} = op; 2081 let Inst{31-26} = 0x3a; //encoding 2082 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2083 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2084 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2085 let Inst{54} = !if(ps.has_slc, slc, ?); 2086 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2087 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2088} 2089 2090//===----------------------------------------------------------------------===// 2091// MTBUF - GFX10. 2092//===----------------------------------------------------------------------===// 2093 2094class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2095 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2096 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 2097 let Inst{25-19} = format; 2098 let Inst{53} = op{3}; 2099} 2100 2101let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2102 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2103 def _BOTHEN_gfx10 : 2104 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2105 def _IDXEN_gfx10 : 2106 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2107 def _OFFEN_gfx10 : 2108 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2109 def _OFFSET_gfx10 : 2110 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2111 } 2112} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2113 2114defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2115defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2116defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2117defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2118defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2119defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2120defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2121defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2122 2123//===----------------------------------------------------------------------===// 2124// MTBUF - GFX6, GFX7, GFX10. 2125//===----------------------------------------------------------------------===// 2126 2127class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2128 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2129 let Inst{15} = ps.addr64; 2130 let Inst{22-19} = dfmt; 2131 let Inst{25-23} = nfmt; 2132} 2133 2134let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2135 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2136 def _ADDR64_gfx6_gfx7 : 2137 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2138 def _BOTHEN_gfx6_gfx7 : 2139 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2140 def _IDXEN_gfx6_gfx7 : 2141 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2142 def _OFFEN_gfx6_gfx7 : 2143 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2144 def _OFFSET_gfx6_gfx7 : 2145 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2146 } 2147} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2148 2149multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2150 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2151 2152defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2153defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2154defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2155defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2156defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2157defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2158defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2159defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2160 2161//===----------------------------------------------------------------------===// 2162// GFX8, GFX9 (VI). 2163//===----------------------------------------------------------------------===// 2164 2165class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 2166 MUBUF_Real<ps>, 2167 Enc64, 2168 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2169 let AssemblerPredicate = isGFX8GFX9; 2170 let DecoderNamespace = "GFX8"; 2171 2172 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2173 let Inst{12} = ps.offen; 2174 let Inst{13} = ps.idxen; 2175 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2176 let Inst{16} = !if(ps.lds, 1, 0); 2177 let Inst{17} = !if(ps.has_slc, slc, ?); 2178 let Inst{24-18} = op; 2179 let Inst{31-26} = 0x38; //encoding 2180 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2181 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2182 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2183 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2184 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2185} 2186 2187multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2188 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2189 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2190 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2191 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2192} 2193 2194multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2195 2196 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2197 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2198 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2199 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2200 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2201 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2202 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2203 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2204 2205 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2206 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2207 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2208 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2209 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2210 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2211 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2212 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2213} 2214 2215class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2216 MUBUF_Real<ps>, 2217 Enc64, 2218 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2219 let AssemblerPredicate=HasUnpackedD16VMem; 2220 let DecoderNamespace="GFX80_UNPACKED"; 2221 2222 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2223 let Inst{12} = ps.offen; 2224 let Inst{13} = ps.idxen; 2225 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2226 let Inst{16} = !if(ps.lds, 1, 0); 2227 let Inst{17} = !if(ps.has_slc, slc, ?); 2228 let Inst{24-18} = op; 2229 let Inst{31-26} = 0x38; //encoding 2230 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2231 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2232 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2233 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2234 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2235} 2236 2237multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2238 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2239 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2240 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2241 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2242} 2243 2244multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2245 MUBUF_Real_AllAddr_vi<op> { 2246 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2247 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2248 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2249 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2250} 2251 2252defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2253defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2254defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2255defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2256defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2257defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2258defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2259defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2260let SubtargetPredicate = HasUnpackedD16VMem in { 2261 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2262 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2263 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2264 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2265 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2266 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2267 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2268 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2269} // End HasUnpackedD16VMem. 2270let SubtargetPredicate = HasPackedD16VMem in { 2271 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2272 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2273 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2274 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2275 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2276 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2277 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2278 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2279} // End HasPackedD16VMem. 2280defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2281defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2282defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2283defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2284defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2285defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2286defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2287defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2288defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2289defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2290defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2291defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2292defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2293defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2294defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2295defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2296 2297defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2298defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2299defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2300defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2301defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2302defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2303 2304defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2305defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2306 2307defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2308defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2309defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2310defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2311defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2312defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2313defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2314defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2315defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2316defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2317defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2318defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2319defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2320 2321defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2322defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2323defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2324defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2325defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2326defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2327defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2328defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2329defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2330defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2331defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2332defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2333defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2334 2335def BUFFER_STORE_LDS_DWORD_vi : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>; 2336 2337def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2338def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2339 2340let SubtargetPredicate = HasAtomicFaddInsts in { 2341 2342defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_AllAddr_vi <0x4d>; 2343defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>; 2344 2345} // End SubtargetPredicate = HasAtomicFaddInsts 2346 2347class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2348 MTBUF_Real<ps>, 2349 Enc64, 2350 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2351 let AssemblerPredicate = isGFX8GFX9; 2352 let DecoderNamespace = "GFX8"; 2353 2354 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2355 let Inst{12} = ps.offen; 2356 let Inst{13} = ps.idxen; 2357 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2358 let Inst{18-15} = op; 2359 let Inst{22-19} = dfmt; 2360 let Inst{25-23} = nfmt; 2361 let Inst{31-26} = 0x3a; //encoding 2362 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2363 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2364 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2365 let Inst{54} = !if(ps.has_slc, slc, ?); 2366 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2367 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2368} 2369 2370multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2371 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2372 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2373 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2374 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2375} 2376 2377class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2378 MTBUF_Real<ps>, 2379 Enc64, 2380 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2381 let AssemblerPredicate=HasUnpackedD16VMem; 2382 let DecoderNamespace="GFX80_UNPACKED"; 2383 2384 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2385 let Inst{12} = ps.offen; 2386 let Inst{13} = ps.idxen; 2387 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2388 let Inst{18-15} = op; 2389 let Inst{22-19} = dfmt; 2390 let Inst{25-23} = nfmt; 2391 let Inst{31-26} = 0x3a; //encoding 2392 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2393 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2394 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2395 let Inst{54} = !if(ps.has_slc, slc, ?); 2396 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2397 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2398} 2399 2400multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2401 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2402 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2403 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2404 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2405} 2406 2407defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2408defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2409defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2410defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2411defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2412defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2413defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2414defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2415let SubtargetPredicate = HasUnpackedD16VMem in { 2416 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2417 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2418 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2419 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2420 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2421 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2422 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2423 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2424} // End HasUnpackedD16VMem. 2425let SubtargetPredicate = HasPackedD16VMem in { 2426 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2427 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2428 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2429 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2430 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2431 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2432 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2433 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2434} // End HasUnpackedD16VMem. 2435 2436def MUBUFInfoTable : GenericTable { 2437 let FilterClass = "MUBUF_Pseudo"; 2438 let CppTypeName = "MUBUFInfo"; 2439 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2440 2441 let PrimaryKey = ["Opcode"]; 2442 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2443} 2444 2445def getMUBUFInfoFromOpcode : SearchIndex { 2446 let Table = MUBUFInfoTable; 2447 let Key = ["Opcode"]; 2448} 2449 2450def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2451 let Table = MUBUFInfoTable; 2452 let Key = ["BaseOpcode", "elements"]; 2453} 2454 2455def MTBUFInfoTable : GenericTable { 2456 let FilterClass = "MTBUF_Pseudo"; 2457 let CppTypeName = "MTBUFInfo"; 2458 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2459 2460 let PrimaryKey = ["Opcode"]; 2461 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2462} 2463 2464def getMTBUFInfoFromOpcode : SearchIndex { 2465 let Table = MTBUFInfoTable; 2466 let Key = ["Opcode"]; 2467} 2468 2469def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2470 let Table = MTBUFInfoTable; 2471 let Key = ["BaseOpcode", "elements"]; 2472} 2473