1/* Copyright (C) 2008-2019 Free Software Foundation, Inc. 2 3 This file is free software; you can redistribute it and/or modify it under 4 the terms of the GNU General Public License as published by the Free 5 Software Foundation; either version 3 of the License, or (at your option) 6 any later version. 7 8 This file is distributed in the hope that it will be useful, but WITHOUT 9 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 for more details. 12 13 Under Section 7 of GPL version 3, you are granted additional 14 permissions described in the GCC Runtime Library Exception, version 15 3.1, as published by the Free Software Foundation. 16 17 You should have received a copy of the GNU General Public License and 18 a copy of the GCC Runtime Library Exception along with this program; 19 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 20 <http://www.gnu.org/licenses/>. */ 21 22#include <spu_intrinsics.h> 23 24typedef int TItype __attribute__ ((mode (TI))); 25 26union qword_TItype 27 { 28 qword q; 29 TItype t; 30 }; 31 32inline static qword 33si_from_TItype (TItype t) 34{ 35 union qword_TItype u; 36 u.t = t; 37 return u.q; 38} 39 40inline static TItype 41si_to_TItype (qword q) 42{ 43 union qword_TItype u; 44 u.q = q; 45 return u.t; 46} 47 48/* A straight forward vectorization and unrolling of 49 * short l[8], r[8]; 50 * TItype total = 0; 51 * for (i = 0; i < 8; i++) 52 * for (j = 0; j < 8; j++) 53 * total += (TItype)((l[7-i] * r[7-j]) << (16 * (i + j))); 54 */ 55TItype 56__multi3 (TItype l, TItype r) 57{ 58 qword u = si_from_TItype (l); 59 qword v = si_from_TItype (r); 60 qword splat0 = si_shufb (v, v, si_ilh (0x0001)); 61 qword splat1 = si_shufb (v, v, si_ilh (0x0203)); 62 qword splat2 = si_shufb (v, v, si_ilh (0x0405)); 63 qword splat3 = si_shufb (v, v, si_ilh (0x0607)); 64 qword splat4 = si_shufb (v, v, si_ilh (0x0809)); 65 qword splat5 = si_shufb (v, v, si_ilh (0x0a0b)); 66 qword splat6 = si_shufb (v, v, si_ilh (0x0c0d)); 67 qword splat7 = si_shufb (v, v, si_ilh (0x0e0f)); 68 69 qword part0l = si_shlqbyi (si_mpyu (u, splat0), 14); 70 qword part1h = si_shlqbyi (si_mpyhhu (u, splat1), 14); 71 qword part1l = si_shlqbyi (si_mpyu (u, splat1), 12); 72 qword part2h = si_shlqbyi (si_mpyhhu (u, splat2), 12); 73 qword part2l = si_shlqbyi (si_mpyu (u, splat2), 10); 74 qword part3h = si_shlqbyi (si_mpyhhu (u, splat3), 10); 75 qword part3l = si_shlqbyi (si_mpyu (u, splat3), 8); 76 qword part4h = si_shlqbyi (si_mpyhhu (u, splat4), 8); 77 qword part4l = si_shlqbyi (si_mpyu (u, splat4), 6); 78 qword part5h = si_shlqbyi (si_mpyhhu (u, splat5), 6); 79 qword part5l = si_shlqbyi (si_mpyu (u, splat5), 4); 80 qword part6h = si_shlqbyi (si_mpyhhu (u, splat6), 4); 81 qword part6l = si_shlqbyi (si_mpyu (u, splat6), 2); 82 qword part7h = si_shlqbyi (si_mpyhhu (u, splat7), 2); 83 qword part7l = si_mpyu (u, splat7); 84 85 qword carry, total0, total1, total2, total3, total4; 86 qword total5, total6, total7, total8, total9, total10; 87 qword total; 88 89 total0 = si_a (si_a (si_a (part0l, part1h), si_a (part1l, part2h)), part7l); 90 total1 = si_a (part2l, part3h); 91 total2 = si_a (part3l, part4h); 92 total3 = si_a (part4l, part5h); 93 total4 = si_a (part5l, part6h); 94 total5 = si_a (part6l, part7h); 95 total6 = si_a (total0, total1); 96 total7 = si_a (total2, total3); 97 total8 = si_a (total4, total5); 98 total9 = si_a (total6, total7); 99 total10 = si_a (total8, total9); 100 101 carry = si_cg (part2l, part3h); 102 carry = si_a (carry, si_cg (part3l, part4h)); 103 carry = si_a (carry, si_cg (part4l, part5h)); 104 carry = si_a (carry, si_cg (part5l, part6h)); 105 carry = si_a (carry, si_cg (part6l, part7h)); 106 carry = si_a (carry, si_cg (total0, total1)); 107 carry = si_a (carry, si_cg (total2, total3)); 108 carry = si_a (carry, si_cg (total4, total5)); 109 carry = si_a (carry, si_cg (total6, total7)); 110 carry = si_a (carry, si_cg (total8, total9)); 111 carry = si_shlqbyi (carry, 4); 112 113 total = si_cg (total10, carry); 114 total = si_shlqbyi (total, 4); 115 total = si_cgx (total10, carry, total); 116 total = si_shlqbyi (total, 4); 117 total = si_addx (total10, carry, total); 118 return si_to_TItype (total); 119} 120