1//===-- lib/fp_lib.h - Floating-point utilities -------------------*- C -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a configuration header for soft-float routines in compiler-rt.
10// This file does not provide any part of the compiler-rt interface, but defines
11// many useful constants and utility routines that are used in the
12// implementation of the soft-float routines in compiler-rt.
13//
14// Assumes that float, double and long double correspond to the IEEE-754
15// binary32, binary64 and binary 128 types, respectively, and that integer
16// endianness matches floating point endianness on the target platform.
17//
18//===----------------------------------------------------------------------===//
19
20#ifndef FP_LIB_HEADER
21#define FP_LIB_HEADER
22
23#include "int_lib.h"
24#include "int_math.h"
25#include <limits.h>
26#include <stdbool.h>
27#include <stdint.h>
28
29// x86_64 FreeBSD prior v9.3 define fixed-width types incorrectly in
30// 32-bit mode.
31#if defined(__FreeBSD__) && defined(__i386__)
32#include <sys/param.h>
33#if __FreeBSD_version < 903000 // v9.3
34#define uint64_t unsigned long long
35#define int64_t long long
36#undef UINT64_C
37#define UINT64_C(c) (c##ULL)
38#endif
39#endif
40
41#if defined SINGLE_PRECISION
42
43typedef uint32_t rep_t;
44typedef int32_t srep_t;
45typedef float fp_t;
46#define REP_C UINT32_C
47#define significandBits 23
48
49static __inline int rep_clz(rep_t a) { return __builtin_clz(a); }
50
51// 32x32 --> 64 bit multiply
52static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {
53  const uint64_t product = (uint64_t)a * b;
54  *hi = product >> 32;
55  *lo = product;
56}
57COMPILER_RT_ABI fp_t __addsf3(fp_t a, fp_t b);
58
59#elif defined DOUBLE_PRECISION
60
61typedef uint64_t rep_t;
62typedef int64_t srep_t;
63typedef double fp_t;
64#define REP_C UINT64_C
65#define significandBits 52
66
67static __inline int rep_clz(rep_t a) {
68#if defined __LP64__
69  return __builtin_clzl(a);
70#else
71  if (a & REP_C(0xffffffff00000000))
72    return __builtin_clz(a >> 32);
73  else
74    return 32 + __builtin_clz(a & REP_C(0xffffffff));
75#endif
76}
77
78#define loWord(a) (a & 0xffffffffU)
79#define hiWord(a) (a >> 32)
80
81// 64x64 -> 128 wide multiply for platforms that don't have such an operation;
82// many 64-bit platforms have this operation, but they tend to have hardware
83// floating-point, so we don't bother with a special case for them here.
84static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {
85  // Each of the component 32x32 -> 64 products
86  const uint64_t plolo = loWord(a) * loWord(b);
87  const uint64_t plohi = loWord(a) * hiWord(b);
88  const uint64_t philo = hiWord(a) * loWord(b);
89  const uint64_t phihi = hiWord(a) * hiWord(b);
90  // Sum terms that contribute to lo in a way that allows us to get the carry
91  const uint64_t r0 = loWord(plolo);
92  const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo);
93  *lo = r0 + (r1 << 32);
94  // Sum terms contributing to hi with the carry from lo
95  *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi;
96}
97#undef loWord
98#undef hiWord
99
100COMPILER_RT_ABI fp_t __adddf3(fp_t a, fp_t b);
101
102#elif defined QUAD_PRECISION
103#if __LDBL_MANT_DIG__ == 113 && defined(__SIZEOF_INT128__)
104#define CRT_LDBL_128BIT
105typedef __uint128_t rep_t;
106typedef __int128_t srep_t;
107typedef long double fp_t;
108#define REP_C (__uint128_t)
109// Note: Since there is no explicit way to tell compiler the constant is a
110// 128-bit integer, we let the constant be casted to 128-bit integer
111#define significandBits 112
112
113static __inline int rep_clz(rep_t a) {
114  const union {
115    __uint128_t ll;
116#if _YUGA_BIG_ENDIAN
117    struct {
118      uint64_t high, low;
119    } s;
120#else
121    struct {
122      uint64_t low, high;
123    } s;
124#endif
125  } uu = {.ll = a};
126
127  uint64_t word;
128  uint64_t add;
129
130  if (uu.s.high) {
131    word = uu.s.high;
132    add = 0;
133  } else {
134    word = uu.s.low;
135    add = 64;
136  }
137  return __builtin_clzll(word) + add;
138}
139
140#define Word_LoMask UINT64_C(0x00000000ffffffff)
141#define Word_HiMask UINT64_C(0xffffffff00000000)
142#define Word_FullMask UINT64_C(0xffffffffffffffff)
143#define Word_1(a) (uint64_t)((a >> 96) & Word_LoMask)
144#define Word_2(a) (uint64_t)((a >> 64) & Word_LoMask)
145#define Word_3(a) (uint64_t)((a >> 32) & Word_LoMask)
146#define Word_4(a) (uint64_t)(a & Word_LoMask)
147
148// 128x128 -> 256 wide multiply for platforms that don't have such an operation;
149// many 64-bit platforms have this operation, but they tend to have hardware
150// floating-point, so we don't bother with a special case for them here.
151static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) {
152
153  const uint64_t product11 = Word_1(a) * Word_1(b);
154  const uint64_t product12 = Word_1(a) * Word_2(b);
155  const uint64_t product13 = Word_1(a) * Word_3(b);
156  const uint64_t product14 = Word_1(a) * Word_4(b);
157  const uint64_t product21 = Word_2(a) * Word_1(b);
158  const uint64_t product22 = Word_2(a) * Word_2(b);
159  const uint64_t product23 = Word_2(a) * Word_3(b);
160  const uint64_t product24 = Word_2(a) * Word_4(b);
161  const uint64_t product31 = Word_3(a) * Word_1(b);
162  const uint64_t product32 = Word_3(a) * Word_2(b);
163  const uint64_t product33 = Word_3(a) * Word_3(b);
164  const uint64_t product34 = Word_3(a) * Word_4(b);
165  const uint64_t product41 = Word_4(a) * Word_1(b);
166  const uint64_t product42 = Word_4(a) * Word_2(b);
167  const uint64_t product43 = Word_4(a) * Word_3(b);
168  const uint64_t product44 = Word_4(a) * Word_4(b);
169
170  const __uint128_t sum0 = (__uint128_t)product44;
171  const __uint128_t sum1 = (__uint128_t)product34 + (__uint128_t)product43;
172  const __uint128_t sum2 =
173      (__uint128_t)product24 + (__uint128_t)product33 + (__uint128_t)product42;
174  const __uint128_t sum3 = (__uint128_t)product14 + (__uint128_t)product23 +
175                           (__uint128_t)product32 + (__uint128_t)product41;
176  const __uint128_t sum4 =
177      (__uint128_t)product13 + (__uint128_t)product22 + (__uint128_t)product31;
178  const __uint128_t sum5 = (__uint128_t)product12 + (__uint128_t)product21;
179  const __uint128_t sum6 = (__uint128_t)product11;
180
181  const __uint128_t r0 = (sum0 & Word_FullMask) + ((sum1 & Word_LoMask) << 32);
182  const __uint128_t r1 = (sum0 >> 64) + ((sum1 >> 32) & Word_FullMask) +
183                         (sum2 & Word_FullMask) + ((sum3 << 32) & Word_HiMask);
184
185  *lo = r0 + (r1 << 64);
186  *hi = (r1 >> 64) + (sum1 >> 96) + (sum2 >> 64) + (sum3 >> 32) + sum4 +
187        (sum5 << 32) + (sum6 << 64);
188}
189#undef Word_1
190#undef Word_2
191#undef Word_3
192#undef Word_4
193#undef Word_HiMask
194#undef Word_LoMask
195#undef Word_FullMask
196#endif // __LDBL_MANT_DIG__ == 113 && __SIZEOF_INT128__
197#else
198#error SINGLE_PRECISION, DOUBLE_PRECISION or QUAD_PRECISION must be defined.
199#endif
200
201#if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) ||                  \
202    defined(CRT_LDBL_128BIT)
203#define typeWidth (sizeof(rep_t) * CHAR_BIT)
204#define exponentBits (typeWidth - significandBits - 1)
205#define maxExponent ((1 << exponentBits) - 1)
206#define exponentBias (maxExponent >> 1)
207
208#define implicitBit (REP_C(1) << significandBits)
209#define significandMask (implicitBit - 1U)
210#define signBit (REP_C(1) << (significandBits + exponentBits))
211#define absMask (signBit - 1U)
212#define exponentMask (absMask ^ significandMask)
213#define oneRep ((rep_t)exponentBias << significandBits)
214#define infRep exponentMask
215#define quietBit (implicitBit >> 1)
216#define qnanRep (exponentMask | quietBit)
217
218static __inline rep_t toRep(fp_t x) {
219  const union {
220    fp_t f;
221    rep_t i;
222  } rep = {.f = x};
223  return rep.i;
224}
225
226static __inline fp_t fromRep(rep_t x) {
227  const union {
228    fp_t f;
229    rep_t i;
230  } rep = {.i = x};
231  return rep.f;
232}
233
234static __inline int normalize(rep_t *significand) {
235  const int shift = rep_clz(*significand) - rep_clz(implicitBit);
236  *significand <<= shift;
237  return 1 - shift;
238}
239
240static __inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) {
241  *hi = *hi << count | *lo >> (typeWidth - count);
242  *lo = *lo << count;
243}
244
245static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo,
246                                              unsigned int count) {
247  if (count < typeWidth) {
248    const bool sticky = (*lo << (typeWidth - count)) != 0;
249    *lo = *hi << (typeWidth - count) | *lo >> count | sticky;
250    *hi = *hi >> count;
251  } else if (count < 2 * typeWidth) {
252    const bool sticky = *hi << (2 * typeWidth - count) | *lo;
253    *lo = *hi >> (count - typeWidth) | sticky;
254    *hi = 0;
255  } else {
256    const bool sticky = *hi | *lo;
257    *lo = sticky;
258    *hi = 0;
259  }
260}
261
262// Implements logb methods (logb, logbf, logbl) for IEEE-754. This avoids
263// pulling in a libm dependency from compiler-rt, but is not meant to replace
264// it (i.e. code calling logb() should get the one from libm, not this), hence
265// the __compiler_rt prefix.
266static __inline fp_t __compiler_rt_logbX(fp_t x) {
267  rep_t rep = toRep(x);
268  int exp = (rep & exponentMask) >> significandBits;
269
270  // Abnormal cases:
271  // 1) +/- inf returns +inf; NaN returns NaN
272  // 2) 0.0 returns -inf
273  if (exp == maxExponent) {
274    if (((rep & signBit) == 0) || (x != x)) {
275      return x; // NaN or +inf: return x
276    } else {
277      return -x; // -inf: return -x
278    }
279  } else if (x == 0.0) {
280    // 0.0: return -inf
281    return fromRep(infRep | signBit);
282  }
283
284  if (exp != 0) {
285    // Normal number
286    return exp - exponentBias; // Unbias exponent
287  } else {
288    // Subnormal number; normalize and repeat
289    rep &= absMask;
290    const int shift = 1 - normalize(&rep);
291    exp = (rep & exponentMask) >> significandBits;
292    return exp - exponentBias - shift; // Unbias exponent
293  }
294}
295#endif
296
297#if defined(SINGLE_PRECISION)
298static __inline fp_t __compiler_rt_logbf(fp_t x) {
299  return __compiler_rt_logbX(x);
300}
301#elif defined(DOUBLE_PRECISION)
302static __inline fp_t __compiler_rt_logb(fp_t x) {
303  return __compiler_rt_logbX(x);
304}
305#elif defined(QUAD_PRECISION)
306#if defined(CRT_LDBL_128BIT)
307static __inline fp_t __compiler_rt_logbl(fp_t x) {
308  return __compiler_rt_logbX(x);
309}
310#else
311// The generic implementation only works for ieee754 floating point. For other
312// floating point types, continue to rely on the libm implementation for now.
313static __inline long double __compiler_rt_logbl(long double x) {
314  return crt_logbl(x);
315}
316#endif
317#endif
318
319#endif // FP_LIB_HEADER
320