1;; Matrix-Multiply Assist (MMA) patterns. 2;; Copyright (C) 2020 Free Software Foundation, Inc. 3;; Contributed by Peter Bergner <bergner@linux.ibm.com> and 4;; Michael Meissner <meissner@linux.ibm.com> 5;; 6;; This file is part of GCC. 7;; 8;; GCC is free software; you can redistribute it and/or modify it 9;; under the terms of the GNU General Public License as published 10;; by the Free Software Foundation; either version 3, or (at your 11;; option) any later version. 12;; 13;; GCC is distributed in the hope that it will be useful, but WITHOUT 14;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 15;; or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public 16;; License for more details. 17;; 18;; You should have received a copy of the GNU General Public License 19;; along with GCC; see the file COPYING3. If not see 20;; <http://www.gnu.org/licenses/>. 21 22;; The MMA patterns use the multi-register PXImode and POImode partial 23;; integer modes to implement the target specific __vector_quad and 24;; __vector_pair types that the MMA built-in functions reference. 25;; To use these modes, we must define XImode and OImode move patterns 26;; so the independent parts of the compiler can use our large partial 27;; integer modes. However, if we enable the XImode and OImode move 28;; patterns, then the compiler will attempt to use them and this can 29;; cause byte swapping issues on litte-endian systems. We don't need 30;; the XImode and OImode move patterns for actual code generation, 31;; therefore, we define the XImode and OImode move patterns, but we 32;; disable their use with a "false" condition flag. 33 34(define_constants [(MAX_MMA_OPERANDS 7)]) 35 36;; Constants for creating unspecs 37 38(define_c_enum "unspec" 39 [UNSPEC_VSX_ASSEMBLE 40 UNSPEC_MMA_ASSEMBLE_ACC 41 UNSPEC_MMA_PMXVBF16GER2 42 UNSPEC_MMA_PMXVBF16GER2NN 43 UNSPEC_MMA_PMXVBF16GER2NP 44 UNSPEC_MMA_PMXVBF16GER2PN 45 UNSPEC_MMA_PMXVBF16GER2PP 46 UNSPEC_MMA_PMXVF16GER2 47 UNSPEC_MMA_PMXVF16GER2NN 48 UNSPEC_MMA_PMXVF16GER2NP 49 UNSPEC_MMA_PMXVF16GER2PN 50 UNSPEC_MMA_PMXVF16GER2PP 51 UNSPEC_MMA_PMXVF32GER 52 UNSPEC_MMA_PMXVF32GERNN 53 UNSPEC_MMA_PMXVF32GERNP 54 UNSPEC_MMA_PMXVF32GERPN 55 UNSPEC_MMA_PMXVF32GERPP 56 UNSPEC_MMA_PMXVF64GER 57 UNSPEC_MMA_PMXVF64GERNN 58 UNSPEC_MMA_PMXVF64GERNP 59 UNSPEC_MMA_PMXVF64GERPN 60 UNSPEC_MMA_PMXVF64GERPP 61 UNSPEC_MMA_PMXVI16GER2 62 UNSPEC_MMA_PMXVI16GER2PP 63 UNSPEC_MMA_PMXVI16GER2S 64 UNSPEC_MMA_PMXVI16GER2SPP 65 UNSPEC_MMA_PMXVI4GER8 66 UNSPEC_MMA_PMXVI4GER8PP 67 UNSPEC_MMA_PMXVI8GER4 68 UNSPEC_MMA_PMXVI8GER4PP 69 UNSPEC_MMA_PMXVI8GER4SPP 70 UNSPEC_MMA_XVBF16GER2 71 UNSPEC_MMA_XVBF16GER2NN 72 UNSPEC_MMA_XVBF16GER2NP 73 UNSPEC_MMA_XVBF16GER2PN 74 UNSPEC_MMA_XVBF16GER2PP 75 UNSPEC_MMA_XVF16GER2 76 UNSPEC_MMA_XVF16GER2NN 77 UNSPEC_MMA_XVF16GER2NP 78 UNSPEC_MMA_XVF16GER2PN 79 UNSPEC_MMA_XVF16GER2PP 80 UNSPEC_MMA_XVF32GER 81 UNSPEC_MMA_XVF32GERNN 82 UNSPEC_MMA_XVF32GERNP 83 UNSPEC_MMA_XVF32GERPN 84 UNSPEC_MMA_XVF32GERPP 85 UNSPEC_MMA_XVF64GER 86 UNSPEC_MMA_XVF64GERNN 87 UNSPEC_MMA_XVF64GERNP 88 UNSPEC_MMA_XVF64GERPN 89 UNSPEC_MMA_XVF64GERPP 90 UNSPEC_MMA_XVI16GER2 91 UNSPEC_MMA_XVI16GER2PP 92 UNSPEC_MMA_XVI16GER2S 93 UNSPEC_MMA_XVI16GER2SPP 94 UNSPEC_MMA_XVI4GER8 95 UNSPEC_MMA_XVI4GER8PP 96 UNSPEC_MMA_XVI8GER4 97 UNSPEC_MMA_XVI8GER4PP 98 UNSPEC_MMA_XVI8GER4SPP 99 UNSPEC_MMA_XXMFACC 100 UNSPEC_MMA_XXMTACC 101 ]) 102 103(define_c_enum "unspecv" 104 [UNSPECV_MMA_XXSETACCZ 105 ]) 106 107;; MMA instructions with 1 accumulator argument 108(define_int_iterator MMA_ACC [UNSPEC_MMA_XXMFACC 109 UNSPEC_MMA_XXMTACC]) 110 111;; MMA instructions with 2 vector arguments 112(define_int_iterator MMA_VV [UNSPEC_MMA_XVI4GER8 113 UNSPEC_MMA_XVI8GER4 114 UNSPEC_MMA_XVI16GER2 115 UNSPEC_MMA_XVI16GER2S 116 UNSPEC_MMA_XVF16GER2 117 UNSPEC_MMA_XVBF16GER2 118 UNSPEC_MMA_XVF32GER]) 119 120;; MMA instructions with 1 accumulator and 2 vector arguments 121(define_int_iterator MMA_AVV [UNSPEC_MMA_XVI4GER8PP 122 UNSPEC_MMA_XVI8GER4PP 123 UNSPEC_MMA_XVI8GER4SPP 124 UNSPEC_MMA_XVI16GER2PP 125 UNSPEC_MMA_XVI16GER2SPP 126 UNSPEC_MMA_XVF16GER2PP 127 UNSPEC_MMA_XVF16GER2PN 128 UNSPEC_MMA_XVF16GER2NP 129 UNSPEC_MMA_XVF16GER2NN 130 UNSPEC_MMA_XVBF16GER2PP 131 UNSPEC_MMA_XVBF16GER2PN 132 UNSPEC_MMA_XVBF16GER2NP 133 UNSPEC_MMA_XVBF16GER2NN 134 UNSPEC_MMA_XVF32GERPP 135 UNSPEC_MMA_XVF32GERPN 136 UNSPEC_MMA_XVF32GERNP 137 UNSPEC_MMA_XVF32GERNN]) 138 139;; MMA instructions with 1 vector pair and 1 vector arguments 140(define_int_iterator MMA_PV [UNSPEC_MMA_XVF64GER]) 141 142;; MMA instructions with 1 accumulator, 1 vector pair and 1 vector arguments 143(define_int_iterator MMA_APV [UNSPEC_MMA_XVF64GERPP 144 UNSPEC_MMA_XVF64GERPN 145 UNSPEC_MMA_XVF64GERNP 146 UNSPEC_MMA_XVF64GERNN]) 147 148;; MMA instructions with 2 vector, 2 4-bit and 1 8-bit arguments 149(define_int_iterator MMA_VVI4I4I8 [UNSPEC_MMA_PMXVI4GER8]) 150 151;; MMA instructions with 1 accumulator, 2 vector, 2 4-bit and 1 8-bit arguments 152(define_int_iterator MMA_AVVI4I4I8 [UNSPEC_MMA_PMXVI4GER8PP]) 153 154;; MMA instructions with 2 vector, 2 4-bit and 1 2-bit arguments 155(define_int_iterator MMA_VVI4I4I2 [UNSPEC_MMA_PMXVI16GER2 156 UNSPEC_MMA_PMXVI16GER2S 157 UNSPEC_MMA_PMXVF16GER2 158 UNSPEC_MMA_PMXVBF16GER2]) 159 160;; MMA instructions with 1 accumulator, 2 vector, 2 4-bit and 1 2-bit arguments 161(define_int_iterator MMA_AVVI4I4I2 [UNSPEC_MMA_PMXVI16GER2PP 162 UNSPEC_MMA_PMXVI16GER2SPP 163 UNSPEC_MMA_PMXVF16GER2PP 164 UNSPEC_MMA_PMXVF16GER2PN 165 UNSPEC_MMA_PMXVF16GER2NP 166 UNSPEC_MMA_PMXVF16GER2NN 167 UNSPEC_MMA_PMXVBF16GER2PP 168 UNSPEC_MMA_PMXVBF16GER2PN 169 UNSPEC_MMA_PMXVBF16GER2NP 170 UNSPEC_MMA_PMXVBF16GER2NN]) 171 172;; MMA instructions with 2 vector and 2 4-bit arguments 173(define_int_iterator MMA_VVI4I4 [UNSPEC_MMA_PMXVF32GER]) 174 175;; MMA instructions with 1 accumulator, 2 vector and 2 4-bit arguments 176(define_int_iterator MMA_AVVI4I4 [UNSPEC_MMA_PMXVF32GERPP 177 UNSPEC_MMA_PMXVF32GERPN 178 UNSPEC_MMA_PMXVF32GERNP 179 UNSPEC_MMA_PMXVF32GERNN]) 180 181;; MMA instructions with 2 vector, 1 4-bit and 1 2-bit arguments 182(define_int_iterator MMA_PVI4I2 [UNSPEC_MMA_PMXVF64GER]) 183 184;; MMA instructions with 1 accumulator, 2 vector, 1 4-bit and 1 2-bit arguments 185(define_int_iterator MMA_APVI4I2 [UNSPEC_MMA_PMXVF64GERPP 186 UNSPEC_MMA_PMXVF64GERPN 187 UNSPEC_MMA_PMXVF64GERNP 188 UNSPEC_MMA_PMXVF64GERNN]) 189 190;; MMA instructions with 2 vector and 3 4-bit arguments 191(define_int_iterator MMA_VVI4I4I4 [UNSPEC_MMA_PMXVI8GER4]) 192 193;; MMA instructions with 1 accumulator, 2 vector and 3 4-bit arguments 194(define_int_iterator MMA_AVVI4I4I4 [UNSPEC_MMA_PMXVI8GER4PP 195 UNSPEC_MMA_PMXVI8GER4SPP]) 196 197(define_int_attr acc [(UNSPEC_MMA_XXMFACC "xxmfacc") 198 (UNSPEC_MMA_XXMTACC "xxmtacc")]) 199 200(define_int_attr vv [(UNSPEC_MMA_XVI4GER8 "xvi4ger8") 201 (UNSPEC_MMA_XVI8GER4 "xvi8ger4") 202 (UNSPEC_MMA_XVI16GER2 "xvi16ger2") 203 (UNSPEC_MMA_XVI16GER2S "xvi16ger2s") 204 (UNSPEC_MMA_XVF16GER2 "xvf16ger2") 205 (UNSPEC_MMA_XVBF16GER2 "xvbf16ger2") 206 (UNSPEC_MMA_XVF32GER "xvf32ger")]) 207 208(define_int_attr avv [(UNSPEC_MMA_XVI4GER8PP "xvi4ger8pp") 209 (UNSPEC_MMA_XVI8GER4PP "xvi8ger4pp") 210 (UNSPEC_MMA_XVI8GER4SPP "xvi8ger4spp") 211 (UNSPEC_MMA_XVI16GER2PP "xvi16ger2pp") 212 (UNSPEC_MMA_XVI16GER2SPP "xvi16ger2spp") 213 (UNSPEC_MMA_XVF16GER2PP "xvf16ger2pp") 214 (UNSPEC_MMA_XVF16GER2PN "xvf16ger2pn") 215 (UNSPEC_MMA_XVF16GER2NP "xvf16ger2np") 216 (UNSPEC_MMA_XVF16GER2NN "xvf16ger2nn") 217 (UNSPEC_MMA_XVBF16GER2PP "xvbf16ger2pp") 218 (UNSPEC_MMA_XVBF16GER2PN "xvbf16ger2pn") 219 (UNSPEC_MMA_XVBF16GER2NP "xvbf16ger2np") 220 (UNSPEC_MMA_XVBF16GER2NN "xvbf16ger2nn") 221 (UNSPEC_MMA_XVF32GERPP "xvf32gerpp") 222 (UNSPEC_MMA_XVF32GERPN "xvf32gerpn") 223 (UNSPEC_MMA_XVF32GERNP "xvf32gernp") 224 (UNSPEC_MMA_XVF32GERNN "xvf32gernn")]) 225 226(define_int_attr pv [(UNSPEC_MMA_XVF64GER "xvf64ger")]) 227 228(define_int_attr apv [(UNSPEC_MMA_XVF64GERPP "xvf64gerpp") 229 (UNSPEC_MMA_XVF64GERPN "xvf64gerpn") 230 (UNSPEC_MMA_XVF64GERNP "xvf64gernp") 231 (UNSPEC_MMA_XVF64GERNN "xvf64gernn")]) 232 233(define_int_attr vvi4i4i8 [(UNSPEC_MMA_PMXVI4GER8 "pmxvi4ger8")]) 234 235(define_int_attr avvi4i4i8 [(UNSPEC_MMA_PMXVI4GER8PP "pmxvi4ger8pp")]) 236 237(define_int_attr vvi4i4i2 [(UNSPEC_MMA_PMXVI16GER2 "pmxvi16ger2") 238 (UNSPEC_MMA_PMXVI16GER2S "pmxvi16ger2s") 239 (UNSPEC_MMA_PMXVF16GER2 "pmxvf16ger2") 240 (UNSPEC_MMA_PMXVBF16GER2 "pmxvbf16ger2")]) 241 242(define_int_attr avvi4i4i2 [(UNSPEC_MMA_PMXVI16GER2PP "pmxvi16ger2pp") 243 (UNSPEC_MMA_PMXVI16GER2SPP "pmxvi16ger2spp") 244 (UNSPEC_MMA_PMXVF16GER2PP "pmxvf16ger2pp") 245 (UNSPEC_MMA_PMXVF16GER2PN "pmxvf16ger2pn") 246 (UNSPEC_MMA_PMXVF16GER2NP "pmxvf16ger2np") 247 (UNSPEC_MMA_PMXVF16GER2NN "pmxvf16ger2nn") 248 (UNSPEC_MMA_PMXVBF16GER2PP "pmxvbf16ger2pp") 249 (UNSPEC_MMA_PMXVBF16GER2PN "pmxvbf16ger2pn") 250 (UNSPEC_MMA_PMXVBF16GER2NP "pmxvbf16ger2np") 251 (UNSPEC_MMA_PMXVBF16GER2NN "pmxvbf16ger2nn")]) 252 253(define_int_attr vvi4i4 [(UNSPEC_MMA_PMXVF32GER "pmxvf32ger")]) 254 255(define_int_attr avvi4i4 [(UNSPEC_MMA_PMXVF32GERPP "pmxvf32gerpp") 256 (UNSPEC_MMA_PMXVF32GERPN "pmxvf32gerpn") 257 (UNSPEC_MMA_PMXVF32GERNP "pmxvf32gernp") 258 (UNSPEC_MMA_PMXVF32GERNN "pmxvf32gernn")]) 259 260(define_int_attr pvi4i2 [(UNSPEC_MMA_PMXVF64GER "pmxvf64ger")]) 261 262(define_int_attr apvi4i2 [(UNSPEC_MMA_PMXVF64GERPP "pmxvf64gerpp") 263 (UNSPEC_MMA_PMXVF64GERPN "pmxvf64gerpn") 264 (UNSPEC_MMA_PMXVF64GERNP "pmxvf64gernp") 265 (UNSPEC_MMA_PMXVF64GERNN "pmxvf64gernn")]) 266 267(define_int_attr vvi4i4i4 [(UNSPEC_MMA_PMXVI8GER4 "pmxvi8ger4")]) 268 269(define_int_attr avvi4i4i4 [(UNSPEC_MMA_PMXVI8GER4PP "pmxvi8ger4pp") 270 (UNSPEC_MMA_PMXVI8GER4SPP "pmxvi8ger4spp")]) 271 272 273;; Define a disabled OImode move pattern, so we can use POImode. 274(define_expand "movoi" 275 [(set (match_operand:OI 0 "nonimmediate_operand") 276 (match_operand:OI 1 "input_operand"))] 277 "0" 278{ 279 gcc_unreachable (); 280}) 281 282;; Vector pair support. POImode can only live in VSRs. 283(define_expand "movpoi" 284 [(set (match_operand:POI 0 "nonimmediate_operand") 285 (match_operand:POI 1 "input_operand"))] 286 "" 287{ 288 if (TARGET_MMA) 289 { 290 rs6000_emit_move (operands[0], operands[1], POImode); 291 DONE; 292 } 293 else if (currently_expanding_to_rtl && seen_error ()) 294 { 295 /* PR103353 shows we may want to continue to expand the __builtin_vsx_lxvp 296 built-in function, even if we have already emitted error messages about 297 some missing required conditions. As shown in that PR, without one 298 explicit mov optab on POImode provided, it would call emit_move_insn 299 recursively. So we allow this pattern to be generated when we are 300 expanding to RTL and have seen errors. It would not cause further ICEs 301 as the compilation would stop soon after expanding. */ 302 } 303 else if (rs6000_opaque_type_invalid_use_p (currently_expanding_gimple_stmt)) 304 ; 305 else 306 /* Catch unexpected cases. */ 307 gcc_assert (false); 308}) 309 310(define_insn_and_split "*movpoi" 311 [(set (match_operand:POI 0 "nonimmediate_operand" "=wa,m,wa") 312 (match_operand:POI 1 "input_operand" "m,wa,wa"))] 313 "TARGET_MMA 314 && (gpc_reg_operand (operands[0], POImode) 315 || gpc_reg_operand (operands[1], POImode))" 316 "@ 317 lxvp%X1 %x0,%1 318 stxvp%X0 %x1,%0 319 #" 320 "&& reload_completed 321 && (!MEM_P (operands[0]) && !MEM_P (operands[1]))" 322 [(const_int 0)] 323{ 324 rs6000_split_multireg_move (operands[0], operands[1]); 325 DONE; 326} 327 [(set_attr "type" "vecload,vecstore,veclogical") 328 (set_attr "length" "*,*,8")]) 329 330 331;; Define a disabled XImode move pattern, so we can use PXImode. 332(define_expand "movxi" 333 [(set (match_operand:XI 0 "nonimmediate_operand") 334 (match_operand:XI 1 "input_operand"))] 335 "0" 336{ 337 gcc_unreachable (); 338}) 339 340;; Vector quad support. PXImode can only live in FPRs. 341(define_expand "movpxi" 342 [(set (match_operand:PXI 0 "nonimmediate_operand") 343 (match_operand:PXI 1 "input_operand"))] 344 "" 345{ 346 if (TARGET_MMA) 347 { 348 rs6000_emit_move (operands[0], operands[1], PXImode); 349 DONE; 350 } 351 else if (currently_expanding_to_rtl && seen_error ()) 352 { 353 /* PR103353 shows we may want to continue to expand the __builtin_vsx_lxvp 354 built-in function, even if we have already emitted error messages about 355 some missing required conditions. So do the same handlings for PXImode 356 as POImode here. */ 357 } 358 else if (rs6000_opaque_type_invalid_use_p (currently_expanding_gimple_stmt)) 359 ; 360 else 361 /* Catch unexpected cases. */ 362 gcc_assert (false); 363}) 364 365(define_insn_and_split "*movpxi" 366 [(set (match_operand:PXI 0 "nonimmediate_operand" "=d,m,d") 367 (match_operand:PXI 1 "input_operand" "m,d,d"))] 368 "TARGET_MMA 369 && (gpc_reg_operand (operands[0], PXImode) 370 || gpc_reg_operand (operands[1], PXImode))" 371 "@ 372 # 373 # 374 #" 375 "&& reload_completed" 376 [(const_int 0)] 377{ 378 rs6000_split_multireg_move (operands[0], operands[1]); 379 DONE; 380} 381 [(set_attr "type" "vecload,vecstore,veclogical") 382 (set_attr "length" "8,8,16") 383 (set_attr "max_prefixed_insns" "2,2,*")]) 384 385(define_expand "vsx_assemble_pair" 386 [(match_operand:POI 0 "vsx_register_operand") 387 (match_operand:V16QI 1 "mma_assemble_input_operand") 388 (match_operand:V16QI 2 "mma_assemble_input_operand")] 389 "TARGET_MMA" 390{ 391 rtx src = gen_rtx_UNSPEC (POImode, 392 gen_rtvec (2, operands[1], operands[2]), 393 UNSPEC_VSX_ASSEMBLE); 394 emit_move_insn (operands[0], src); 395 DONE; 396}) 397 398;; We cannot update the two output registers atomically, so mark the output 399;; as an early clobber so we don't accidentally clobber the input operands. */ 400 401(define_insn_and_split "*vsx_assemble_pair" 402 [(set (match_operand:POI 0 "vsx_register_operand" "=&wa") 403 (unspec:POI [(match_operand:V16QI 1 "mma_assemble_input_operand" "mwa") 404 (match_operand:V16QI 2 "mma_assemble_input_operand" "mwa")] 405 UNSPEC_VSX_ASSEMBLE))] 406 "TARGET_MMA" 407 "#" 408 "&& reload_completed" 409 [(const_int 0)] 410{ 411 rtx src = gen_rtx_UNSPEC (POImode, 412 gen_rtvec (2, operands[1], operands[2]), 413 UNSPEC_VSX_ASSEMBLE); 414 rs6000_split_multireg_move (operands[0], src); 415 DONE; 416}) 417 418(define_expand "mma_assemble_acc" 419 [(match_operand:PXI 0 "fpr_reg_operand") 420 (match_operand:V16QI 1 "input_operand") 421 (match_operand:V16QI 2 "input_operand") 422 (match_operand:V16QI 3 "input_operand") 423 (match_operand:V16QI 4 "input_operand")] 424 "TARGET_MMA" 425{ 426 rtx src = gen_rtx_UNSPEC (PXImode, 427 gen_rtvec (4, operands[1], operands[2], 428 operands[3], operands[4]), 429 UNSPEC_MMA_ASSEMBLE_ACC); 430 emit_move_insn (operands[0], src); 431 DONE; 432}) 433 434;; We cannot update the four output registers atomically, so mark the output 435;; as an early clobber so we don't accidentally clobber the input operands. */ 436 437(define_insn_and_split "*mma_assemble_acc" 438 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 439 (unspec:PXI [(match_operand:V16QI 1 "mma_assemble_input_operand" "mwa") 440 (match_operand:V16QI 2 "mma_assemble_input_operand" "mwa") 441 (match_operand:V16QI 3 "mma_assemble_input_operand" "mwa") 442 (match_operand:V16QI 4 "mma_assemble_input_operand" "mwa")] 443 UNSPEC_MMA_ASSEMBLE_ACC))] 444 "TARGET_MMA 445 && fpr_reg_operand (operands[0], PXImode)" 446 "#" 447 "&& reload_completed" 448 [(const_int 0)] 449{ 450 rtx src = gen_rtx_UNSPEC (PXImode, 451 gen_rtvec (4, operands[1], operands[2], 452 operands[3], operands[4]), 453 UNSPEC_MMA_ASSEMBLE_ACC); 454 rs6000_split_multireg_move (operands[0], src); 455 DONE; 456}) 457 458;; MMA instructions that do not use their accumulators as an input, still 459;; must not allow their vector operands to overlap the registers used by 460;; the accumulator. We enforce this by marking the output as early clobber. 461 462(define_insn "mma_<acc>" 463 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 464 (unspec:PXI [(match_operand:PXI 1 "fpr_reg_operand" "0")] 465 MMA_ACC))] 466 "TARGET_MMA" 467 "<acc> %A0" 468 [(set_attr "type" "mma")]) 469 470;; We can't have integer constants in PXImode so we wrap this in an 471;; UNSPEC_VOLATILE. 472 473(define_insn "mma_xxsetaccz" 474 [(set (match_operand:PXI 0 "fpr_reg_operand" "=d") 475 (unspec_volatile:PXI [(const_int 0)] 476 UNSPECV_MMA_XXSETACCZ))] 477 "TARGET_MMA" 478 "xxsetaccz %A0" 479 [(set_attr "type" "mma")]) 480 481(define_insn "mma_<vv>" 482 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 483 (unspec:PXI [(match_operand:V16QI 1 "vsx_register_operand" "wa") 484 (match_operand:V16QI 2 "vsx_register_operand" "wa")] 485 MMA_VV))] 486 "TARGET_MMA" 487 "<vv> %A0,%x1,%x2" 488 [(set_attr "type" "mma")]) 489 490(define_insn "mma_<avv>" 491 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 492 (unspec:PXI [(match_operand:PXI 1 "fpr_reg_operand" "0") 493 (match_operand:V16QI 2 "vsx_register_operand" "wa") 494 (match_operand:V16QI 3 "vsx_register_operand" "wa")] 495 MMA_AVV))] 496 "TARGET_MMA" 497 "<avv> %A0,%x2,%x3" 498 [(set_attr "type" "mma")]) 499 500(define_insn "mma_<pv>" 501 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 502 (unspec:PXI [(match_operand:POI 1 "vsx_register_operand" "wa") 503 (match_operand:V16QI 2 "vsx_register_operand" "wa")] 504 MMA_PV))] 505 "TARGET_MMA" 506 "<pv> %A0,%x1,%x2" 507 [(set_attr "type" "mma")]) 508 509(define_insn "mma_<apv>" 510 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 511 (unspec:PXI [(match_operand:PXI 1 "fpr_reg_operand" "0") 512 (match_operand:POI 2 "vsx_register_operand" "wa") 513 (match_operand:V16QI 3 "vsx_register_operand" "wa")] 514 MMA_APV))] 515 "TARGET_MMA" 516 "<apv> %A0,%x2,%x3" 517 [(set_attr "type" "mma")]) 518 519(define_insn "mma_<vvi4i4i8>" 520 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 521 (unspec:PXI [(match_operand:V16QI 1 "vsx_register_operand" "wa") 522 (match_operand:V16QI 2 "vsx_register_operand" "wa") 523 (match_operand:SI 3 "const_0_to_15_operand" "n") 524 (match_operand:SI 4 "const_0_to_15_operand" "n") 525 (match_operand:SI 5 "u8bit_cint_operand" "n")] 526 MMA_VVI4I4I8))] 527 "TARGET_MMA" 528 "<vvi4i4i8> %A0,%x1,%x2,%3,%4,%5" 529 [(set_attr "type" "mma") 530 (set_attr "length" "8")]) 531 532(define_insn "mma_<avvi4i4i8>" 533 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 534 (unspec:PXI [(match_operand:PXI 1 "fpr_reg_operand" "0") 535 (match_operand:V16QI 2 "vsx_register_operand" "wa") 536 (match_operand:V16QI 3 "vsx_register_operand" "wa") 537 (match_operand:SI 4 "const_0_to_15_operand" "n") 538 (match_operand:SI 5 "const_0_to_15_operand" "n") 539 (match_operand:SI 6 "u8bit_cint_operand" "n")] 540 MMA_AVVI4I4I8))] 541 "TARGET_MMA" 542 "<avvi4i4i8> %A0,%x2,%x3,%4,%5,%6" 543 [(set_attr "type" "mma") 544 (set_attr "length" "8")]) 545 546(define_insn "mma_<vvi4i4i2>" 547 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 548 (unspec:PXI [(match_operand:V16QI 1 "vsx_register_operand" "wa") 549 (match_operand:V16QI 2 "vsx_register_operand" "wa") 550 (match_operand:SI 3 "const_0_to_15_operand" "n") 551 (match_operand:SI 4 "const_0_to_15_operand" "n") 552 (match_operand:SI 5 "const_0_to_3_operand" "n")] 553 MMA_VVI4I4I2))] 554 "TARGET_MMA" 555 "<vvi4i4i2> %A0,%x1,%x2,%3,%4,%5" 556 [(set_attr "type" "mma") 557 (set_attr "length" "8")]) 558 559(define_insn "mma_<avvi4i4i2>" 560 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 561 (unspec:PXI [(match_operand:PXI 1 "fpr_reg_operand" "0") 562 (match_operand:V16QI 2 "vsx_register_operand" "wa") 563 (match_operand:V16QI 3 "vsx_register_operand" "wa") 564 (match_operand:SI 4 "const_0_to_15_operand" "n") 565 (match_operand:SI 5 "const_0_to_15_operand" "n") 566 (match_operand:SI 6 "const_0_to_3_operand" "n")] 567 MMA_AVVI4I4I2))] 568 "TARGET_MMA" 569 "<avvi4i4i2> %A0,%x2,%x3,%4,%5,%6" 570 [(set_attr "type" "mma") 571 (set_attr "length" "8")]) 572 573(define_insn "mma_<vvi4i4>" 574 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 575 (unspec:PXI [(match_operand:V16QI 1 "vsx_register_operand" "wa") 576 (match_operand:V16QI 2 "vsx_register_operand" "wa") 577 (match_operand:SI 3 "const_0_to_15_operand" "n") 578 (match_operand:SI 4 "const_0_to_15_operand" "n")] 579 MMA_VVI4I4))] 580 "TARGET_MMA" 581 "<vvi4i4> %A0,%x1,%x2,%3,%4" 582 [(set_attr "type" "mma") 583 (set_attr "length" "8")]) 584 585(define_insn "mma_<avvi4i4>" 586 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 587 (unspec:PXI [(match_operand:PXI 1 "fpr_reg_operand" "0") 588 (match_operand:V16QI 2 "vsx_register_operand" "wa") 589 (match_operand:V16QI 3 "vsx_register_operand" "wa") 590 (match_operand:SI 4 "const_0_to_15_operand" "n") 591 (match_operand:SI 5 "const_0_to_15_operand" "n")] 592 MMA_AVVI4I4))] 593 "TARGET_MMA" 594 "<avvi4i4> %A0,%x2,%x3,%4,%5" 595 [(set_attr "type" "mma") 596 (set_attr "length" "8")]) 597 598(define_insn "mma_<pvi4i2>" 599 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 600 (unspec:PXI [(match_operand:POI 1 "vsx_register_operand" "wa") 601 (match_operand:V16QI 2 "vsx_register_operand" "wa") 602 (match_operand:SI 3 "const_0_to_15_operand" "n") 603 (match_operand:SI 4 "const_0_to_3_operand" "n")] 604 MMA_PVI4I2))] 605 "TARGET_MMA" 606 "<pvi4i2> %A0,%x1,%x2,%3,%4" 607 [(set_attr "type" "mma") 608 (set_attr "length" "8")]) 609 610(define_insn "mma_<apvi4i2>" 611 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 612 (unspec:PXI [(match_operand:PXI 1 "fpr_reg_operand" "0") 613 (match_operand:POI 2 "vsx_register_operand" "wa") 614 (match_operand:V16QI 3 "vsx_register_operand" "wa") 615 (match_operand:SI 4 "const_0_to_15_operand" "n") 616 (match_operand:SI 5 "const_0_to_3_operand" "n")] 617 MMA_APVI4I2))] 618 "TARGET_MMA" 619 "<apvi4i2> %A0,%x2,%x3,%4,%5" 620 [(set_attr "type" "mma") 621 (set_attr "length" "8")]) 622 623(define_insn "mma_<vvi4i4i4>" 624 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 625 (unspec:PXI [(match_operand:V16QI 1 "vsx_register_operand" "wa") 626 (match_operand:V16QI 2 "vsx_register_operand" "wa") 627 (match_operand:SI 3 "const_0_to_15_operand" "n") 628 (match_operand:SI 4 "const_0_to_15_operand" "n") 629 (match_operand:SI 5 "const_0_to_15_operand" "n")] 630 MMA_VVI4I4I4))] 631 "TARGET_MMA" 632 "<vvi4i4i4> %A0,%x1,%x2,%3,%4,%5" 633 [(set_attr "type" "mma") 634 (set_attr "length" "8")]) 635 636(define_insn "mma_<avvi4i4i4>" 637 [(set (match_operand:PXI 0 "fpr_reg_operand" "=&d") 638 (unspec:PXI [(match_operand:PXI 1 "fpr_reg_operand" "0") 639 (match_operand:V16QI 2 "vsx_register_operand" "wa") 640 (match_operand:V16QI 3 "vsx_register_operand" "wa") 641 (match_operand:SI 4 "const_0_to_15_operand" "n") 642 (match_operand:SI 5 "const_0_to_15_operand" "n") 643 (match_operand:SI 6 "const_0_to_15_operand" "n")] 644 MMA_AVVI4I4I4))] 645 "TARGET_MMA" 646 "<avvi4i4i4> %A0,%x2,%x3,%4,%5,%6" 647 [(set_attr "type" "mma") 648 (set_attr "length" "8")]) 649