| #ifndef FENV_PRIVATE_H |
| #define FENV_PRIVATE_H 1 |
| |
| #include <fenv.h> |
| #include <fpu_control.h> |
| |
| #ifdef __SSE2_MATH__ |
| # define math_opt_barrier(x) \ |
| ({ __typeof(x) __x; \ |
| if (sizeof (x) <= sizeof (double)) \ |
| __asm ("" : "=x" (__x) : "0" (x)); \ |
| else \ |
| __asm ("" : "=t" (__x) : "0" (x)); \ |
| __x; }) |
| # define math_force_eval(x) \ |
| do { \ |
| if (sizeof (x) <= sizeof (double)) \ |
| __asm __volatile ("" : : "x" (x)); \ |
| else \ |
| __asm __volatile ("" : : "f" (x)); \ |
| } while (0) |
| #else |
| # define math_opt_barrier(x) \ |
| ({ __typeof (x) __x; \ |
| __asm ("" : "=t" (__x) : "0" (x)); \ |
| __x; }) |
| # define math_force_eval(x) \ |
| do { \ |
| __typeof (x) __x = (x); \ |
| if (sizeof (x) <= sizeof (double)) \ |
| __asm __volatile ("" : : "m" (__x)); \ |
| else \ |
| __asm __volatile ("" : : "f" (__x)); \ |
| } while (0) |
| #endif |
| |
| /* This file is used by both the 32- and 64-bit ports. The 64-bit port |
| has a field in the fenv_t for the mxcsr; the 32-bit port does not. |
| Instead, we (ab)use the only 32-bit field extant in the struct. */ |
| #ifndef __x86_64__ |
| # define __mxcsr __eip |
| #endif |
| |
| |
| /* All of these functions are private to libm, and are all used in pairs |
| to save+change the fp state and restore the original state. Thus we |
| need not care for both the 387 and the sse unit, only the one we're |
| actually using. */ |
| |
| #if defined __AVX__ || defined SSE2AVX |
| # define STMXCSR "vstmxcsr" |
| # define LDMXCSR "vldmxcsr" |
| #else |
| # define STMXCSR "stmxcsr" |
| # define LDMXCSR "ldmxcsr" |
| #endif |
| |
| static __always_inline void |
| libc_feholdexcept_sse (fenv_t *e) |
| { |
| unsigned int mxcsr; |
| asm (STMXCSR " %0" : "=m" (*&mxcsr)); |
| e->__mxcsr = mxcsr; |
| mxcsr = (mxcsr | 0x1f80) & ~0x3f; |
| asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); |
| } |
| |
| static __always_inline void |
| libc_feholdexcept_387 (fenv_t *e) |
| { |
| /* Recall that fnstenv has a side-effect of masking exceptions. |
| Clobber all of the fp registers so that the TOS field is 0. */ |
| asm volatile ("fnstenv %0; fnclex" |
| : "=m"(*e) |
| : : "st", "st(1)", "st(2)", "st(3)", |
| "st(4)", "st(5)", "st(6)", "st(7)"); |
| } |
| |
| static __always_inline void |
| libc_fesetround_sse (int r) |
| { |
| unsigned int mxcsr; |
| asm (STMXCSR " %0" : "=m" (*&mxcsr)); |
| mxcsr = (mxcsr & ~0x6000) | (r << 3); |
| asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); |
| } |
| |
| static __always_inline void |
| libc_fesetround_387 (int r) |
| { |
| fpu_control_t cw; |
| _FPU_GETCW (cw); |
| cw = (cw & ~0xc00) | r; |
| _FPU_SETCW (cw); |
| } |
| |
| static __always_inline void |
| libc_feholdexcept_setround_sse (fenv_t *e, int r) |
| { |
| unsigned int mxcsr; |
| asm (STMXCSR " %0" : "=m" (*&mxcsr)); |
| e->__mxcsr = mxcsr; |
| mxcsr = ((mxcsr | 0x1f80) & ~0x603f) | (r << 3); |
| asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); |
| } |
| |
| /* Set both rounding mode and precision. A convenience function for use |
| by libc_feholdexcept_setround and libc_feholdexcept_setround_53bit. */ |
| static __always_inline void |
| libc_feholdexcept_setround_387_prec (fenv_t *e, int r) |
| { |
| libc_feholdexcept_387 (e); |
| |
| fpu_control_t cw = e->__control_word; |
| cw &= ~(_FPU_RC_ZERO | _FPU_EXTENDED); |
| cw |= r | 0x3f; |
| _FPU_SETCW (cw); |
| } |
| |
| static __always_inline void |
| libc_feholdexcept_setround_387 (fenv_t *e, int r) |
| { |
| libc_feholdexcept_setround_387_prec (e, r | _FPU_EXTENDED); |
| } |
| |
| static __always_inline void |
| libc_feholdexcept_setround_387_53bit (fenv_t *e, int r) |
| { |
| libc_feholdexcept_setround_387_prec (e, r | _FPU_DOUBLE); |
| } |
| |
| static __always_inline int |
| libc_fetestexcept_sse (int e) |
| { |
| unsigned int mxcsr; |
| asm volatile (STMXCSR " %0" : "=m" (*&mxcsr)); |
| return mxcsr & e & FE_ALL_EXCEPT; |
| } |
| |
| static __always_inline int |
| libc_fetestexcept_387 (int ex) |
| { |
| fexcept_t temp; |
| asm volatile ("fnstsw %0" : "=a" (temp)); |
| return temp & ex & FE_ALL_EXCEPT; |
| } |
| |
| static __always_inline void |
| libc_fesetenv_sse (fenv_t *e) |
| { |
| asm volatile (LDMXCSR " %0" : : "m" (e->__mxcsr)); |
| } |
| |
| static __always_inline void |
| libc_fesetenv_387 (fenv_t *e) |
| { |
| /* Clobber all fp registers so that the TOS value we saved earlier is |
| compatible with the current state of the compiler. */ |
| asm volatile ("fldenv %0" |
| : : "m" (*e) |
| : "st", "st(1)", "st(2)", "st(3)", |
| "st(4)", "st(5)", "st(6)", "st(7)"); |
| } |
| |
| static __always_inline int |
| libc_feupdateenv_test_sse (fenv_t *e, int ex) |
| { |
| unsigned int mxcsr, old_mxcsr, cur_ex; |
| asm volatile (STMXCSR " %0" : "=m" (*&mxcsr)); |
| cur_ex = mxcsr & FE_ALL_EXCEPT; |
| |
| /* Merge current exceptions with the old environment. */ |
| old_mxcsr = e->__mxcsr; |
| mxcsr = old_mxcsr | cur_ex; |
| asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); |
| |
| /* Raise SIGFPE for any new exceptions since the hold. Expect that |
| the normal environment has all exceptions masked. */ |
| if (__glibc_unlikely (~(old_mxcsr >> 7) & cur_ex)) |
| __feraiseexcept (cur_ex); |
| |
| /* Test for exceptions raised since the hold. */ |
| return cur_ex & ex; |
| } |
| |
| static __always_inline int |
| libc_feupdateenv_test_387 (fenv_t *e, int ex) |
| { |
| fexcept_t cur_ex; |
| |
| /* Save current exceptions. */ |
| asm volatile ("fnstsw %0" : "=a" (cur_ex)); |
| cur_ex &= FE_ALL_EXCEPT; |
| |
| /* Reload original environment. */ |
| libc_fesetenv_387 (e); |
| |
| /* Merge current exceptions. */ |
| __feraiseexcept (cur_ex); |
| |
| /* Test for exceptions raised since the hold. */ |
| return cur_ex & ex; |
| } |
| |
| static __always_inline void |
| libc_feupdateenv_sse (fenv_t *e) |
| { |
| libc_feupdateenv_test_sse (e, 0); |
| } |
| |
| static __always_inline void |
| libc_feupdateenv_387 (fenv_t *e) |
| { |
| libc_feupdateenv_test_387 (e, 0); |
| } |
| |
| static __always_inline void |
| libc_feholdsetround_sse (fenv_t *e, int r) |
| { |
| unsigned int mxcsr; |
| asm (STMXCSR " %0" : "=m" (*&mxcsr)); |
| e->__mxcsr = mxcsr; |
| mxcsr = (mxcsr & ~0x6000) | (r << 3); |
| asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); |
| } |
| |
| static __always_inline void |
| libc_feholdsetround_387_prec (fenv_t *e, int r) |
| { |
| fpu_control_t cw; |
| |
| _FPU_GETCW (cw); |
| e->__control_word = cw; |
| cw &= ~(_FPU_RC_ZERO | _FPU_EXTENDED); |
| cw |= r; |
| _FPU_SETCW (cw); |
| } |
| |
| static __always_inline void |
| libc_feholdsetround_387 (fenv_t *e, int r) |
| { |
| libc_feholdsetround_387_prec (e, r | _FPU_EXTENDED); |
| } |
| |
| static __always_inline void |
| libc_feholdsetround_387_53bit (fenv_t *e, int r) |
| { |
| libc_feholdsetround_387_prec (e, r | _FPU_DOUBLE); |
| } |
| |
| static __always_inline void |
| libc_feresetround_sse (fenv_t *e) |
| { |
| unsigned int mxcsr; |
| asm (STMXCSR " %0" : "=m" (*&mxcsr)); |
| mxcsr = (mxcsr & ~0x6000) | (e->__mxcsr & 0x6000); |
| asm volatile (LDMXCSR " %0" : : "m" (*&mxcsr)); |
| } |
| |
| static __always_inline void |
| libc_feresetround_387 (fenv_t *e) |
| { |
| _FPU_SETCW (e->__control_word); |
| } |
| |
| #ifdef __SSE_MATH__ |
| # define libc_feholdexceptf libc_feholdexcept_sse |
| # define libc_fesetroundf libc_fesetround_sse |
| # define libc_feholdexcept_setroundf libc_feholdexcept_setround_sse |
| # define libc_fetestexceptf libc_fetestexcept_sse |
| # define libc_fesetenvf libc_fesetenv_sse |
| # define libc_feupdateenv_testf libc_feupdateenv_test_sse |
| # define libc_feupdateenvf libc_feupdateenv_sse |
| # define libc_feholdsetroundf libc_feholdsetround_sse |
| # define libc_feresetroundf libc_feresetround_sse |
| #else |
| # define libc_feholdexceptf libc_feholdexcept_387 |
| # define libc_fesetroundf libc_fesetround_387 |
| # define libc_feholdexcept_setroundf libc_feholdexcept_setround_387 |
| # define libc_fetestexceptf libc_fetestexcept_387 |
| # define libc_fesetenvf libc_fesetenv_387 |
| # define libc_feupdateenv_testf libc_feupdateenv_test_387 |
| # define libc_feupdateenvf libc_feupdateenv_387 |
| # define libc_feholdsetroundf libc_feholdsetround_387 |
| # define libc_feresetroundf libc_feresetround_387 |
| #endif /* __SSE_MATH__ */ |
| |
| #ifdef __SSE2_MATH__ |
| # define libc_feholdexcept libc_feholdexcept_sse |
| # define libc_fesetround libc_fesetround_sse |
| # define libc_feholdexcept_setround libc_feholdexcept_setround_sse |
| # define libc_fetestexcept libc_fetestexcept_sse |
| # define libc_fesetenv libc_fesetenv_sse |
| # define libc_feupdateenv_test libc_feupdateenv_test_sse |
| # define libc_feupdateenv libc_feupdateenv_sse |
| # define libc_feholdsetround libc_feholdsetround_sse |
| # define libc_feresetround libc_feresetround_sse |
| #else |
| # define libc_feholdexcept libc_feholdexcept_387 |
| # define libc_fesetround libc_fesetround_387 |
| # define libc_feholdexcept_setround libc_feholdexcept_setround_387 |
| # define libc_fetestexcept libc_fetestexcept_387 |
| # define libc_fesetenv libc_fesetenv_387 |
| # define libc_feupdateenv_test libc_feupdateenv_test_387 |
| # define libc_feupdateenv libc_feupdateenv_387 |
| # define libc_feholdsetround libc_feholdsetround_387 |
| # define libc_feresetround libc_feresetround_387 |
| #endif /* __SSE2_MATH__ */ |
| |
| #define libc_feholdexceptl libc_feholdexcept_387 |
| #define libc_fesetroundl libc_fesetround_387 |
| #define libc_feholdexcept_setroundl libc_feholdexcept_setround_387 |
| #define libc_fetestexceptl libc_fetestexcept_387 |
| #define libc_fesetenvl libc_fesetenv_387 |
| #define libc_feupdateenv_testl libc_feupdateenv_test_387 |
| #define libc_feupdateenvl libc_feupdateenv_387 |
| #define libc_feholdsetroundl libc_feholdsetround_387 |
| #define libc_feresetroundl libc_feresetround_387 |
| |
| #ifndef __SSE2_MATH__ |
| # define libc_feholdexcept_setround_53bit libc_feholdexcept_setround_387_53bit |
| # define libc_feholdsetround_53bit libc_feholdsetround_387_53bit |
| #endif |
| |
| /* We have support for rounding mode context. */ |
| #define HAVE_RM_CTX 1 |
| |
| static __always_inline void |
| libc_feholdexcept_setround_sse_ctx (struct rm_ctx *ctx, int r) |
| { |
| unsigned int mxcsr, new_mxcsr; |
| asm (STMXCSR " %0" : "=m" (*&mxcsr)); |
| new_mxcsr = ((mxcsr | 0x1f80) & ~0x603f) | (r << 3); |
| |
| ctx->env.__mxcsr = mxcsr; |
| if (__glibc_unlikely (mxcsr != new_mxcsr)) |
| { |
| asm volatile (LDMXCSR " %0" : : "m" (*&new_mxcsr)); |
| ctx->updated_status = true; |
| } |
| else |
| ctx->updated_status = false; |
| } |
| |
| /* Unconditional since we want to overwrite any exceptions that occurred in the |
| context. This is also why all fehold* functions unconditionally write into |
| ctx->env. */ |
| static __always_inline void |
| libc_fesetenv_sse_ctx (struct rm_ctx *ctx) |
| { |
| libc_fesetenv_sse (&ctx->env); |
| } |
| |
| static __always_inline void |
| libc_feupdateenv_sse_ctx (struct rm_ctx *ctx) |
| { |
| if (__glibc_unlikely (ctx->updated_status)) |
| libc_feupdateenv_test_sse (&ctx->env, 0); |
| } |
| |
| static __always_inline void |
| libc_feholdexcept_setround_387_prec_ctx (struct rm_ctx *ctx, int r) |
| { |
| libc_feholdexcept_387 (&ctx->env); |
| |
| fpu_control_t cw = ctx->env.__control_word; |
| fpu_control_t old_cw = cw; |
| cw &= ~(_FPU_RC_ZERO | _FPU_EXTENDED); |
| cw |= r | 0x3f; |
| |
| if (__glibc_unlikely (old_cw != cw)) |
| { |
| _FPU_SETCW (cw); |
| ctx->updated_status = true; |
| } |
| else |
| ctx->updated_status = false; |
| } |
| |
| static __always_inline void |
| libc_feholdexcept_setround_387_ctx (struct rm_ctx *ctx, int r) |
| { |
| libc_feholdexcept_setround_387_prec_ctx (ctx, r | _FPU_EXTENDED); |
| } |
| |
| static __always_inline void |
| libc_feholdexcept_setround_387_53bit_ctx (struct rm_ctx *ctx, int r) |
| { |
| libc_feholdexcept_setround_387_prec_ctx (ctx, r | _FPU_DOUBLE); |
| } |
| |
| static __always_inline void |
| libc_feholdsetround_387_prec_ctx (struct rm_ctx *ctx, int r) |
| { |
| fpu_control_t cw, new_cw; |
| |
| _FPU_GETCW (cw); |
| new_cw = cw; |
| new_cw &= ~(_FPU_RC_ZERO | _FPU_EXTENDED); |
| new_cw |= r; |
| |
| ctx->env.__control_word = cw; |
| if (__glibc_unlikely (new_cw != cw)) |
| { |
| _FPU_SETCW (new_cw); |
| ctx->updated_status = true; |
| } |
| else |
| ctx->updated_status = false; |
| } |
| |
| static __always_inline void |
| libc_feholdsetround_387_ctx (struct rm_ctx *ctx, int r) |
| { |
| libc_feholdsetround_387_prec_ctx (ctx, r | _FPU_EXTENDED); |
| } |
| |
| static __always_inline void |
| libc_feholdsetround_387_53bit_ctx (struct rm_ctx *ctx, int r) |
| { |
| libc_feholdsetround_387_prec_ctx (ctx, r | _FPU_DOUBLE); |
| } |
| |
| static __always_inline void |
| libc_feholdsetround_sse_ctx (struct rm_ctx *ctx, int r) |
| { |
| unsigned int mxcsr, new_mxcsr; |
| |
| asm (STMXCSR " %0" : "=m" (*&mxcsr)); |
| new_mxcsr = (mxcsr & ~0x6000) | (r << 3); |
| |
| ctx->env.__mxcsr = mxcsr; |
| if (__glibc_unlikely (new_mxcsr != mxcsr)) |
| { |
| asm volatile (LDMXCSR " %0" : : "m" (*&new_mxcsr)); |
| ctx->updated_status = true; |
| } |
| else |
| ctx->updated_status = false; |
| } |
| |
| static __always_inline void |
| libc_feresetround_sse_ctx (struct rm_ctx *ctx) |
| { |
| if (__glibc_unlikely (ctx->updated_status)) |
| libc_feresetround_sse (&ctx->env); |
| } |
| |
| static __always_inline void |
| libc_feresetround_387_ctx (struct rm_ctx *ctx) |
| { |
| if (__glibc_unlikely (ctx->updated_status)) |
| _FPU_SETCW (ctx->env.__control_word); |
| } |
| |
| static __always_inline void |
| libc_feupdateenv_387_ctx (struct rm_ctx *ctx) |
| { |
| if (__glibc_unlikely (ctx->updated_status)) |
| libc_feupdateenv_test_387 (&ctx->env, 0); |
| } |
| |
| #ifdef __SSE_MATH__ |
| # define libc_feholdexcept_setroundf_ctx libc_feholdexcept_setround_sse_ctx |
| # define libc_fesetenvf_ctx libc_fesetenv_sse_ctx |
| # define libc_feupdateenvf_ctx libc_feupdateenv_sse_ctx |
| # define libc_feholdsetroundf_ctx libc_feholdsetround_sse_ctx |
| # define libc_feresetroundf_ctx libc_feresetround_sse_ctx |
| #else |
| # define libc_feholdexcept_setroundf_ctx libc_feholdexcept_setround_387_ctx |
| # define libc_feupdateenvf_ctx libc_feupdateenv_387_ctx |
| # define libc_feholdsetroundf_ctx libc_feholdsetround_387_ctx |
| # define libc_feresetroundf_ctx libc_feresetround_387_ctx |
| #endif /* __SSE_MATH__ */ |
| |
| #ifdef __SSE2_MATH__ |
| # define libc_feholdexcept_setround_ctx libc_feholdexcept_setround_sse_ctx |
| # define libc_fesetenv_ctx libc_fesetenv_sse_ctx |
| # define libc_feupdateenv_ctx libc_feupdateenv_sse_ctx |
| # define libc_feholdsetround_ctx libc_feholdsetround_sse_ctx |
| # define libc_feresetround_ctx libc_feresetround_sse_ctx |
| #else |
| # define libc_feholdexcept_setround_ctx libc_feholdexcept_setround_387_ctx |
| # define libc_feupdateenv_ctx libc_feupdateenv_387_ctx |
| # define libc_feresetround_ctx libc_feresetround_387_ctx |
| #endif /* __SSE2_MATH__ */ |
| |
| #define libc_feholdexcept_setroundl_ctx libc_feholdexcept_setround_387_ctx |
| #define libc_feupdateenvl_ctx libc_feupdateenv_387_ctx |
| #define libc_feholdsetroundl_ctx libc_feholdsetround_387_ctx |
| #define libc_feresetroundl_ctx libc_feresetround_387_ctx |
| |
| #ifndef __SSE2_MATH__ |
| # define libc_feholdsetround_53bit_ctx libc_feholdsetround_387_53bit_ctx |
| # define libc_feresetround_53bit_ctx libc_feresetround_387_ctx |
| #endif |
| |
| #undef __mxcsr |
| |
| #endif /* FENV_PRIVATE_H */ |