Update Eigen to commit:f02856c6406cd36b5d7f1b98e27d85df5af08d06 CHANGELOG ========= f02856c64 - Use EIGEN_NOT_A_MACRO macro (oh the irony!) to avoid build issue in TensorFlow. 690ae9502 - Use C++11 standard features for detecting presence of Inf and NaN d71ac6a75 - Fix recent PowerPC warnings and clang warning d54d228b4 - Limit the number of build jobs to 8 and link jobs to 4 for PowerPC. This should help reduce the OOM build problems. 23e154186 - Put deadcode checks back in from previous change. 6c58f0fe1 - Revert changes that made BF16 GEMM to cause bad register spillage for LLVM (Power) PiperOrigin-RevId: 517015735 Change-Id: I8d46a00c6c1993473ae55b6301198b43a635ef97
diff --git a/Eigen/src/Core/MathFunctions.h b/Eigen/src/Core/MathFunctions.h index ecc6f38..40ee3f5 100644 --- a/Eigen/src/Core/MathFunctions.h +++ b/Eigen/src/Core/MathFunctions.h
@@ -229,63 +229,6 @@ typedef typename NumTraits<Scalar>::Real & type; }; - -/**************************************************************************** -* Implementation of sign * -****************************************************************************/ -template<typename Scalar, bool IsComplex = (NumTraits<Scalar>::IsComplex!=0), - bool IsInteger = (NumTraits<Scalar>::IsInteger!=0)> -struct sign_impl -{ - EIGEN_DEVICE_FUNC - static inline Scalar run(const Scalar& a) - { - return Scalar( (a>Scalar(0)) - (a<Scalar(0)) ); - } -}; - -template<typename Scalar> -struct sign_impl<Scalar, false, false> -{ - EIGEN_DEVICE_FUNC - static inline Scalar run(const Scalar& a) - { - return (std::isnan)(a) ? a : Scalar( (a>Scalar(0)) - (a<Scalar(0)) ); - } -}; - -template<typename Scalar, bool IsInteger> -struct sign_impl<Scalar, true, IsInteger> -{ - EIGEN_DEVICE_FUNC - static inline Scalar run(const Scalar& a) - { - using real_type = typename NumTraits<Scalar>::Real; - real_type aa = std::abs(a); - if (aa==real_type(0)) - return Scalar(0); - aa = real_type(1)/aa; - return Scalar(a.real()*aa, a.imag()*aa ); - } -}; - -// The sign function for bool is the identity. -template<> -struct sign_impl<bool, false, true> -{ - EIGEN_DEVICE_FUNC - static inline bool run(const bool& a) - { - return a; - } -}; - -template<typename Scalar> -struct sign_retval -{ - typedef Scalar type; -}; - /**************************************************************************** * Implementation of conj * ****************************************************************************/ @@ -664,7 +607,7 @@ EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) { EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) - using std::expm1; + EIGEN_USING_STD(expm1); return expm1(x); } }; @@ -725,7 +668,7 @@ EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) { - using std::log1p; + EIGEN_USING_STD(log1p); return log1p(x); } }; @@ -938,128 +881,114 @@ // Implementation of is* functions -// std::is* do not work with fast-math and gcc, std::is* are available on MSVC 2013 and newer, as well as in clang. -#if (!(EIGEN_COMP_GNUC_STRICT && __FINITE_MATH_ONLY__)) || (EIGEN_COMP_MSVC) || (EIGEN_COMP_CLANG) -#define EIGEN_USE_STD_FPCLASSIFY 1 -#else -#define EIGEN_USE_STD_FPCLASSIFY 0 -#endif - -template<typename T> -EIGEN_DEVICE_FUNC -std::enable_if_t<internal::is_integral<T>::value,bool> -isnan_impl(const T&) { return false; } - -template<typename T> -EIGEN_DEVICE_FUNC -std::enable_if_t<internal::is_integral<T>::value,bool> -isinf_impl(const T&) { return false; } - -template<typename T> -EIGEN_DEVICE_FUNC -std::enable_if_t<internal::is_integral<T>::value,bool> -isfinite_impl(const T&) { return true; } - -template<typename T> -EIGEN_DEVICE_FUNC -std::enable_if_t<(!internal::is_integral<T>::value)&&(!NumTraits<T>::IsComplex),bool> -isfinite_impl(const T& x) -{ - #if defined(EIGEN_GPU_COMPILE_PHASE) - return (::isfinite)(x); - #elif EIGEN_USE_STD_FPCLASSIFY - using std::isfinite; - return isfinite EIGEN_NOT_A_MACRO (x); - #else - return x<=NumTraits<T>::highest() && x>=NumTraits<T>::lowest(); - #endif +template <typename T> +EIGEN_DEVICE_FUNC std::enable_if_t<!(std::numeric_limits<T>::has_infinity || + std::numeric_limits<T>::has_quiet_NaN || + std::numeric_limits<T>::has_signaling_NaN), + bool> +isfinite_impl(const T&) { + return true; } -template<typename T> +template <typename T> +EIGEN_DEVICE_FUNC std::enable_if_t<(std::numeric_limits<T>::has_infinity || std::numeric_limits<T>::has_quiet_NaN || + std::numeric_limits<T>::has_signaling_NaN) && + (!NumTraits<T>::IsComplex), + bool> +isfinite_impl(const T& x) { + EIGEN_USING_STD(isfinite); + return isfinite EIGEN_NOT_A_MACRO (x); +} + +template <typename T> +EIGEN_DEVICE_FUNC std::enable_if_t<!std::numeric_limits<T>::has_infinity, bool> +isinf_impl(const T&) { + return false; +} + +template <typename T> +EIGEN_DEVICE_FUNC std::enable_if_t< + (std::numeric_limits<T>::has_infinity && !NumTraits<T>::IsComplex), bool> +isinf_impl(const T& x) { + EIGEN_USING_STD(isinf); + return isinf EIGEN_NOT_A_MACRO (x); +} + +template <typename T> +EIGEN_DEVICE_FUNC std::enable_if_t<!(std::numeric_limits<T>::has_quiet_NaN || + std::numeric_limits<T>::has_signaling_NaN), + bool> +isnan_impl(const T&) { + return false; +} + +template <typename T> EIGEN_DEVICE_FUNC -std::enable_if_t<(!internal::is_integral<T>::value)&&(!NumTraits<T>::IsComplex),bool> -isinf_impl(const T& x) -{ - #if defined(EIGEN_GPU_COMPILE_PHASE) - return (::isinf)(x); - #elif EIGEN_USE_STD_FPCLASSIFY - using std::isinf; - return isinf EIGEN_NOT_A_MACRO (x); - #else - return x>NumTraits<T>::highest() || x<NumTraits<T>::lowest(); - #endif + std::enable_if_t<(std::numeric_limits<T>::has_quiet_NaN || + std::numeric_limits<T>::has_signaling_NaN) && + (!NumTraits<T>::IsComplex), + bool> + isnan_impl(const T& x) { + EIGEN_USING_STD(isnan); + return isnan EIGEN_NOT_A_MACRO (x); } -template<typename T> -EIGEN_DEVICE_FUNC -std::enable_if_t<(!internal::is_integral<T>::value)&&(!NumTraits<T>::IsComplex),bool> -isnan_impl(const T& x) -{ - #if defined(EIGEN_GPU_COMPILE_PHASE) - return (::isnan)(x); - #elif EIGEN_USE_STD_FPCLASSIFY - using std::isnan; - return isnan EIGEN_NOT_A_MACRO (x); - #else - return x != x; - #endif -} - -#if (!EIGEN_USE_STD_FPCLASSIFY) - -#if EIGEN_COMP_MSVC - -template<typename T> EIGEN_DEVICE_FUNC bool isinf_msvc_helper(T x) -{ - return _fpclass(x)==_FPCLASS_NINF || _fpclass(x)==_FPCLASS_PINF; -} - -//MSVC defines a _isnan builtin function, but for double only -#ifndef EIGEN_GPU_COMPILE_PHASE -EIGEN_DEVICE_FUNC inline bool isnan_impl(const long double& x) { return _isnan(x)!=0; } -#endif -EIGEN_DEVICE_FUNC inline bool isnan_impl(const double& x) { return _isnan(x)!=0; } -EIGEN_DEVICE_FUNC inline bool isnan_impl(const float& x) { return _isnan(x)!=0; } - -#ifndef EIGEN_GPU_COMPILE_PHASE -EIGEN_DEVICE_FUNC inline bool isinf_impl(const long double& x) { return isinf_msvc_helper(x); } -#endif -EIGEN_DEVICE_FUNC inline bool isinf_impl(const double& x) { return isinf_msvc_helper(x); } -EIGEN_DEVICE_FUNC inline bool isinf_impl(const float& x) { return isinf_msvc_helper(x); } - -#elif (defined __FINITE_MATH_ONLY__ && __FINITE_MATH_ONLY__ && EIGEN_COMP_GNUC) - -#if EIGEN_COMP_GNUC - #define EIGEN_TMP_NOOPT_ATTRIB EIGEN_DEVICE_FUNC inline __attribute__((optimize("no-finite-math-only"))) -#else - // NOTE the inline qualifier and noinline attribute are both needed: the former is to avoid linking issue (duplicate symbol), - // while the second prevent too aggressive optimizations in fast-math mode: - #define EIGEN_TMP_NOOPT_ATTRIB EIGEN_DEVICE_FUNC inline __attribute__((noinline,optimize("no-finite-math-only"))) -#endif - -#ifndef EIGEN_GPU_COMPILE_PHASE -template<> EIGEN_TMP_NOOPT_ATTRIB bool isnan_impl(const long double& x) { return __builtin_isnan(x); } -#endif -template<> EIGEN_TMP_NOOPT_ATTRIB bool isnan_impl(const double& x) { return __builtin_isnan(x); } -template<> EIGEN_TMP_NOOPT_ATTRIB bool isnan_impl(const float& x) { return __builtin_isnan(x); } -template<> EIGEN_TMP_NOOPT_ATTRIB bool isinf_impl(const double& x) { return __builtin_isinf(x); } -template<> EIGEN_TMP_NOOPT_ATTRIB bool isinf_impl(const float& x) { return __builtin_isinf(x); } -#ifndef EIGEN_GPU_COMPILE_PHASE -template<> EIGEN_TMP_NOOPT_ATTRIB bool isinf_impl(const long double& x) { return __builtin_isinf(x); } -#endif - -#undef EIGEN_TMP_NOOPT_ATTRIB - -#endif - -#endif - // The following overload are defined at the end of this file -template<typename T> EIGEN_DEVICE_FUNC bool isfinite_impl(const std::complex<T>& x); -template<typename T> EIGEN_DEVICE_FUNC bool isnan_impl(const std::complex<T>& x); -template<typename T> EIGEN_DEVICE_FUNC bool isinf_impl(const std::complex<T>& x); +template <typename T> +EIGEN_DEVICE_FUNC bool isfinite_impl(const std::complex<T>& x); +template <typename T> +EIGEN_DEVICE_FUNC bool isnan_impl(const std::complex<T>& x); +template <typename T> +EIGEN_DEVICE_FUNC bool isinf_impl(const std::complex<T>& x); +template <typename T> +T generic_fast_tanh_float(const T& a_x); -template<typename T> T generic_fast_tanh_float(const T& a_x); +/**************************************************************************** + * Implementation of sign * + ****************************************************************************/ +template <typename Scalar, bool IsComplex = (NumTraits<Scalar>::IsComplex != 0), + bool IsInteger = (NumTraits<Scalar>::IsInteger != 0)> +struct sign_impl { + EIGEN_DEVICE_FUNC + static inline Scalar run(const Scalar& a) { + return Scalar((a > Scalar(0)) - (a < Scalar(0))); + } +}; + +template <typename Scalar> +struct sign_impl<Scalar, false, false> { + EIGEN_DEVICE_FUNC + static inline Scalar run(const Scalar& a) { + return (isnan_impl<Scalar>)(a) ? a + : Scalar((a > Scalar(0)) - (a < Scalar(0))); + } +}; + +template <typename Scalar, bool IsInteger> +struct sign_impl<Scalar, true, IsInteger> { + EIGEN_DEVICE_FUNC + static inline Scalar run(const Scalar& a) { + using real_type = typename NumTraits<Scalar>::Real; + EIGEN_USING_STD(abs); + real_type aa = abs(a); + if (aa == real_type(0)) return Scalar(0); + aa = real_type(1) / aa; + return Scalar(a.real() * aa, a.imag() * aa); + } +}; + +// The sign function for bool is the identity. +template <> +struct sign_impl<bool, false, true> { + EIGEN_DEVICE_FUNC + static inline bool run(const bool& a) { return a; } +}; + +template <typename Scalar> +struct sign_retval { + typedef Scalar type; +}; + } // end namespace internal /****************************************************************************
diff --git a/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h b/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h index 4774587..fe4906d 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h +++ b/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h
@@ -57,10 +57,10 @@ } BFLOAT16_UNROLL - for(Index i = 0, k = 0; i < num_rhs; i++) { + for(Index i = 0, x = 0; i < num_rhs; i++) { BFLOAT16_UNROLL - for(Index j = 0; j < num_lhs; j++, k++) { - __builtin_mma_xvbf16ger2pp(&(quad_acc[k]), reinterpret_cast<Packet16uc>(rhs[i].m_val), reinterpret_cast<Packet16uc>(lhs[j].m_val)); + for(Index j = 0; j < num_lhs; j++, x++) { + __builtin_mma_xvbf16ger2pp(&(quad_acc[x]), reinterpret_cast<Packet16uc>(rhs[i].m_val), reinterpret_cast<Packet16uc>(lhs[j].m_val)); } } } @@ -146,8 +146,8 @@ zeroAccumulators<num_acc>(quad_acc); - Index k = 0; - for(Index j = depth >> 1; j--; k += 2){ + Index k; + for(k = 0; k + 2 <= depth; k += 2){ KLoop<num_acc, num_packets, false, rhsExtraCols, lhsExtraRows, num_rhs, num_lhs>(indexA, indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows); } if(depth&1){ @@ -415,7 +415,7 @@ template<const Index size, bool inc> EIGEN_ALWAYS_INLINE void convertPointerF32toBF16(Index& i, float* result, Index rows, bfloat16*& dst, Index resInc) { - for(Index j = (rows - i) / size; j--; i += size, dst += size*resInc){ + for(; i + size <= rows; i += size, dst += size*resInc){ PacketBlock<Packet8bf,(size+4)/8> r32; r32.packet[0] = convertF32toBF16<size != 4>(result + i + 0); if (size >= 16) { @@ -569,12 +569,11 @@ zeroAccumulators<num_acc>(quad_acc); LhsMapper lhs2 = lhs.getSubMapper(row, 0); - Index j = 0; - for(Index k = cend >> 1; k--; j += 2) { + for(Index j = 0; j + 2 <= cend; j += 2) { vecColLoop<num_acc, LhsMapper, RhsMapper, false>(j, lhs2, rhs, quad_acc); } if (cend & 1) { - vecColLoop<num_acc, LhsMapper, RhsMapper, true>(j, lhs2, rhs, quad_acc); + vecColLoop<num_acc, LhsMapper, RhsMapper, true>(cend - 1, lhs2, rhs, quad_acc); } disassembleAccumulators<num_acc>(quad_acc, acc); @@ -731,12 +730,8 @@ } else { if (extra == 3) { pstoreu_partial(result + k, d0, extra); - } else if (extra == 2) { - Packet2ul d1 = reinterpret_cast<Packet2ul>(d0); - *(unsigned long long *)(result + k) = d1[0]; } else { - Packet4i d1 = reinterpret_cast<Packet4i>(d0); - *(unsigned int *)(result + k) = d1[0]; + memcpy((void *)(result + k), (void *)(&d0), sizeof(float) * extra); } } } @@ -769,7 +764,7 @@ EIGEN_ALWAYS_INLINE void vecLoop(Index cols, const LhsMapper& lhs, RhsMapper& rhs, __vector_quad (&quad_acc)[num_acc], Index extra_cols) { Index j = 0; - for(Index k = cols >> 3; k--; j += 8) { + for(; j + 8 <= cols; j += 8){ multVecLoop<num_acc, LhsMapper, RhsMapper, false>(quad_acc, lhs, rhs, j, extra_cols); } @@ -803,30 +798,38 @@ } while(multiIters && (num_acc <= rows - (row += num_acc))); } +template<const Index num_acc, typename LhsMapper, typename RhsMapper> +EIGEN_ALWAYS_INLINE void colVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) +{ + if (MAX_BFLOAT16_VEC_ACC > num_acc) { + colVecLoopBody<num_acc, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + } +} + template<typename LhsMapper, typename RhsMapper> EIGEN_ALWAYS_INLINE void colVecLoopBodyExtra(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, float *result) { switch (rows - row) { case 7: - colVecLoopBody<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + colVecLoopBodyExtraN<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); break; case 6: - colVecLoopBody<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + colVecLoopBodyExtraN<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); break; case 5: - colVecLoopBody<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + colVecLoopBodyExtraN<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); break; case 4: - colVecLoopBody<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + colVecLoopBodyExtraN<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); break; case 3: - colVecLoopBody<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + colVecLoopBodyExtraN<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); break; case 2: - colVecLoopBody<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + colVecLoopBodyExtraN<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); break; case 1: - colVecLoopBody<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + colVecLoopBodyExtraN<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); break; } }
diff --git a/Eigen/src/Core/util/DisableStupidWarnings.h b/Eigen/src/Core/util/DisableStupidWarnings.h index eed2397..bed6cdd 100644 --- a/Eigen/src/Core/util/DisableStupidWarnings.h +++ b/Eigen/src/Core/util/DisableStupidWarnings.h
@@ -48,7 +48,7 @@ #if __has_warning("-Wimplicit-int-float-conversion") #pragma clang diagnostic ignored "-Wimplicit-int-float-conversion" #endif - #if ( defined(__ALTIVEC__) || defined(__VSX__) ) && __cplusplus < 201103L + #if ( defined(__ALTIVEC__) || defined(__VSX__) ) && ( !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 201112L) ) // warning: generic selections are a C11-specific feature // ignoring warnings thrown at vec_ctf in Altivec/PacketMath.h #if __has_warning("-Wc11-extensions")
diff --git a/test/AnnoyingScalar.h b/test/AnnoyingScalar.h index dc20d43..94b513c 100644 --- a/test/AnnoyingScalar.h +++ b/test/AnnoyingScalar.h
@@ -65,7 +65,7 @@ AnnoyingScalar operator-() const { return AnnoyingScalar(-*v); } - + AnnoyingScalar operator-(const AnnoyingScalar& other) const { return AnnoyingScalar(*v-*other.v); } @@ -140,11 +140,6 @@ bool (isfinite)(const AnnoyingScalar& x) { return (numext::isfinite)(*x.v); } -template<> -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE -bool (isnan)(const AnnoyingScalar& x) { - return (numext::isnan)(*x.v); -} } namespace internal {