Update Eigen to commit:81044ec13df7608d0d9d86aff2ef9805fc69bed1 CHANGELOG ========= 81044ec13 - Provide macro to explicitly disable alloca bcce88c99 - Faster emulated half comparisons ac6955ebc - Remove MSVC warnings in FindCoeff.h 67a898a07 - Fix unprotected SIZE in macro. cdf6a1f5e - Add OpenBLAS sbgemm. d228bcdf8 - Fix neon compilation bug 994f3d107 - Fix neon packet math tests, add missing neon intrinsics cda19a625 - Make `Eigen::Map<const Vector>::operator[]` return correct type PiperOrigin-RevId: 774907248 Change-Id: Ic572084cfda6bbbe9bb0f5ec94da1ea07671e3e4
diff --git a/Eigen/src/Core/DenseCoeffsBase.h b/Eigen/src/Core/DenseCoeffsBase.h index cff104c..377df57 100644 --- a/Eigen/src/Core/DenseCoeffsBase.h +++ b/Eigen/src/Core/DenseCoeffsBase.h
@@ -45,10 +45,16 @@ // - This is the return type of the coeff() method. // - The LvalueBit means exactly that we can offer a coeffRef() method, which means exactly that we can get references // to coeffs, which means exactly that we can have coeff() return a const reference (as opposed to returning a value). + // - The DirectAccessBit means exactly that the underlying data of coefficients can be directly accessed as a plain + // strided array, which means exactly that the underlying data of coefficients does exist in memory, which means + // exactly that the coefficients is const-referencable, which means exactly that we can have coeff() return a const + // reference. For example, Map<const Matrix> have DirectAccessBit but not LvalueBit, so that Map<const Matrix>.coeff() + // does points to a const Scalar& which exists in memory, while does not allow coeffRef() as it would not provide a + // lvalue. Notice that DirectAccessBit and LvalueBit are mutually orthogonal. // - The is_arithmetic check is required since "const int", "const double", etc. will cause warnings on some systems // while the declaration of "const T", where T is a non arithmetic type does not. Always returning "const Scalar&" is // not possible, since the underlying expressions might not offer a valid address the reference could be referring to. - typedef std::conditional_t<bool(internal::traits<Derived>::Flags& LvalueBit), const Scalar&, + typedef std::conditional_t<bool(internal::traits<Derived>::Flags&(LvalueBit | DirectAccessBit)), const Scalar&, std::conditional_t<internal::is_arithmetic<Scalar>::value, Scalar, const Scalar>> CoeffReturnType;
diff --git a/Eigen/src/Core/FindCoeff.h b/Eigen/src/Core/FindCoeff.h index 4569256..0102e8a 100644 --- a/Eigen/src/Core/FindCoeff.h +++ b/Eigen/src/Core/FindCoeff.h
@@ -277,7 +277,7 @@ using Scalar = typename Derived::Scalar; using Packet = typename packet_traits<Scalar>::type; static constexpr int Flags = Base::Flags; - static constexpr bool IsRowMajor = Flags & RowMajorBit; + static constexpr bool IsRowMajor = bool(Flags & RowMajorBit); EIGEN_DEVICE_FUNC inline find_coeff_evaluator(const Derived& xpr) : Base(xpr), m_xpr(xpr) {} EIGEN_DEVICE_FUNC inline Scalar coeffByOuterInner(Index outer, Index inner) const { @@ -313,7 +313,7 @@ using Packet = typename Evaluator::Packet; static constexpr int PacketSize = unpacket_traits<Packet>::size; - static constexpr bool Linearize = Flags & LinearAccessBit; + static constexpr bool Linearize = bool(Flags & LinearAccessBit); static constexpr bool DontVectorize = enum_lt_not_dynamic(Linearize ? MaxSizeAtCompileTime : MaxInnerSizeAtCompileTime, PacketSize); static constexpr bool Vectorize =
diff --git a/Eigen/src/Core/arch/AVX/PacketMath.h b/Eigen/src/Core/arch/AVX/PacketMath.h index 1b1d326..eb5da53 100644 --- a/Eigen/src/Core/arch/AVX/PacketMath.h +++ b/Eigen/src/Core/arch/AVX/PacketMath.h
@@ -2250,23 +2250,63 @@ } template <> +EIGEN_STRONG_INLINE Packet8h pisinf<Packet8h>(const Packet8h& a) { + constexpr uint16_t kInf = ((1 << 5) - 1) << 10; + constexpr uint16_t kAbsMask = (1 << 15) - 1; + return _mm_cmpeq_epi16(_mm_and_si128(a.m_val, _mm_set1_epi16(kAbsMask)), _mm_set1_epi16(kInf)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pisnan<Packet8h>(const Packet8h& a) { + constexpr uint16_t kInf = ((1 << 5) - 1) << 10; + constexpr uint16_t kAbsMask = (1 << 15) - 1; + return _mm_cmpgt_epi16(_mm_and_si128(a.m_val, _mm_set1_epi16(kAbsMask)), _mm_set1_epi16(kInf)); +} + +// convert the sign-magnitude representation to two's complement +EIGEN_STRONG_INLINE __m128i pmaptosigned(const __m128i& a) { + constexpr uint16_t kAbsMask = (1 << 15) - 1; + // if 'a' has the sign bit set, clear the sign bit and negate the result as if it were an integer + return _mm_sign_epi16(_mm_and_si128(a, _mm_set1_epi16(kAbsMask)), a); +} + +// return true if both `a` and `b` are not NaN +EIGEN_STRONG_INLINE Packet8h pisordered(const Packet8h& a, const Packet8h& b) { + constexpr uint16_t kInf = ((1 << 5) - 1) << 10; + constexpr uint16_t kAbsMask = (1 << 15) - 1; + __m128i abs_a = _mm_and_si128(a.m_val, _mm_set1_epi16(kAbsMask)); + __m128i abs_b = _mm_and_si128(b.m_val, _mm_set1_epi16(kAbsMask)); + // check if both `abs_a <= kInf` and `abs_b <= kInf` by checking if max(abs_a, abs_b) <= kInf + // SSE has no `lesser or equal` instruction for integers, but comparing against kInf + 1 accomplishes the same goal + return _mm_cmplt_epi16(_mm_max_epu16(abs_a, abs_b), _mm_set1_epi16(kInf + 1)); +} + +template <> EIGEN_STRONG_INLINE Packet8h pcmp_eq(const Packet8h& a, const Packet8h& b) { - return Pack16To8(pcmp_eq(half2float(a), half2float(b))); + __m128i isOrdered = pisordered(a, b); + __m128i isEqual = _mm_cmpeq_epi16(pmaptosigned(a.m_val), pmaptosigned(b.m_val)); + return _mm_and_si128(isOrdered, isEqual); } template <> EIGEN_STRONG_INLINE Packet8h pcmp_le(const Packet8h& a, const Packet8h& b) { - return Pack16To8(pcmp_le(half2float(a), half2float(b))); + __m128i isOrdered = pisordered(a, b); + __m128i isGreater = _mm_cmpgt_epi16(pmaptosigned(a.m_val), pmaptosigned(b.m_val)); + return _mm_andnot_si128(isGreater, isOrdered); } template <> EIGEN_STRONG_INLINE Packet8h pcmp_lt(const Packet8h& a, const Packet8h& b) { - return Pack16To8(pcmp_lt(half2float(a), half2float(b))); + __m128i isOrdered = pisordered(a, b); + __m128i isLess = _mm_cmplt_epi16(pmaptosigned(a.m_val), pmaptosigned(b.m_val)); + return _mm_and_si128(isOrdered, isLess); } template <> EIGEN_STRONG_INLINE Packet8h pcmp_lt_or_nan(const Packet8h& a, const Packet8h& b) { - return Pack16To8(pcmp_lt_or_nan(half2float(a), half2float(b))); + __m128i isUnordered = por(pisnan(a), pisnan(b)); + __m128i isLess = _mm_cmplt_epi16(pmaptosigned(a.m_val), pmaptosigned(b.m_val)); + return _mm_or_si128(isUnordered, isLess); } template <>
diff --git a/Eigen/src/Core/arch/Default/Half.h b/Eigen/src/Core/arch/Default/Half.h index ba70d5f..c073fe8 100644 --- a/Eigen/src/Core/arch/Default/Half.h +++ b/Eigen/src/Core/arch/Default/Half.h
@@ -497,16 +497,56 @@ a = half(float(a) / float(b)); return a; } + +// Non-negative floating point numbers have a monotonic mapping to non-negative integers. +// This property allows floating point numbers to be reinterpreted as integers for comparisons, which is useful if there +// is no native floating point comparison operator. Floating point signedness is handled by the sign-magnitude +// representation, whereas integers typically use two's complement. Converting the bit pattern from sign-magnitude to +// two's complement allows the transformed bit patterns be compared as signed integers. All edge cases (+/-0 and +/- +// infinity) are handled automatically, except NaN. +// +// fp16 uses 1 sign bit, 5 exponent bits, and 10 mantissa bits. The bit pattern conveys NaN when all the exponent +// bits (5) are set, and at least one mantissa bit is set. The sign bit is irrelevant for determining NaN. To check for +// NaN, clear the sign bit and check if the integral representation is greater than 01111100000000. To test +// for non-NaN, clear the sign bit and check if the integeral representation is less than or equal to 01111100000000. + +// convert sign-magnitude representation to two's complement +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC int16_t mapToSigned(uint16_t a) { + constexpr uint16_t kAbsMask = (1 << 15) - 1; + // If the sign bit is set, clear the sign bit and return the (integer) negation. Otherwise, return the input. + return (a >> 15) ? -(a & kAbsMask) : a; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool isOrdered(const half& a, const half& b) { + constexpr uint16_t kInf = ((1 << 5) - 1) << 10; + constexpr uint16_t kAbsMask = (1 << 15) - 1; + return numext::maxi(a.x & kAbsMask, b.x & kAbsMask) <= kInf; +} EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator==(const half& a, const half& b) { - return numext::equal_strict(float(a), float(b)); + bool result = mapToSigned(a.x) == mapToSigned(b.x); + result &= isOrdered(a, b); + return result; } -EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator!=(const half& a, const half& b) { - return numext::not_equal_strict(float(a), float(b)); +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator!=(const half& a, const half& b) { return !(a == b); } +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<(const half& a, const half& b) { + bool result = mapToSigned(a.x) < mapToSigned(b.x); + result &= isOrdered(a, b); + return result; } -EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<(const half& a, const half& b) { return float(a) < float(b); } -EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<=(const half& a, const half& b) { return float(a) <= float(b); } -EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>(const half& a, const half& b) { return float(a) > float(b); } -EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>=(const half& a, const half& b) { return float(a) >= float(b); } +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<=(const half& a, const half& b) { + bool result = mapToSigned(a.x) <= mapToSigned(b.x); + result &= isOrdered(a, b); + return result; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>(const half& a, const half& b) { + bool result = mapToSigned(a.x) > mapToSigned(b.x); + result &= isOrdered(a, b); + return result; +} +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>=(const half& a, const half& b) { + bool result = mapToSigned(a.x) >= mapToSigned(b.x); + result &= isOrdered(a, b); + return result; +} #if EIGEN_COMP_CLANG && defined(EIGEN_GPUCC) #pragma pop_macro("EIGEN_DEVICE_FUNC") @@ -706,7 +746,11 @@ #endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isfinite)(const half& a) { - return !(isinf EIGEN_NOT_A_MACRO(a)) && !(isnan EIGEN_NOT_A_MACRO(a)); +#if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) || defined(EIGEN_HAS_BUILTIN_FLOAT16) + return (numext::bit_cast<numext::uint16_t>(a.x) & 0x7fff) < 0x7c00; +#else + return (a.x & 0x7fff) < 0x7c00; +#endif } EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half abs(const half& a) {
diff --git a/Eigen/src/Core/arch/NEON/PacketMath.h b/Eigen/src/Core/arch/NEON/PacketMath.h index 6d7f038..9364cff 100644 --- a/Eigen/src/Core/arch/NEON/PacketMath.h +++ b/Eigen/src/Core/arch/NEON/PacketMath.h
@@ -1287,6 +1287,14 @@ EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) { return vfma_f32(c, a, b); } +template <> +EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { + return vfmsq_f32(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2f pnmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) { + return vfms_f32(c, a, b); +} #else template <> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { @@ -1296,7 +1304,31 @@ EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) { return vmla_f32(c, a, b); } +template <> +EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { + return vmlsq_f32(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2f pnmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) { + return vmls_f32(c, a, b); +} #endif +template <> +EIGEN_STRONG_INLINE Packet4f pmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) { + return pnegate(pnmadd(a, b, c)); +} +template <> +EIGEN_STRONG_INLINE Packet2f pmsub(const Packet2f& a, const Packet2f& b, const Packet2f& c) { + return pnegate(pnmadd(a, b, c)); +} +template <> +EIGEN_STRONG_INLINE Packet4f pnmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) { + return pnegate(pmadd(a, b, c)); +} +template <> +EIGEN_STRONG_INLINE Packet2f pnmsub(const Packet2f& a, const Packet2f& b, const Packet2f& c) { + return pnegate(pmadd(a, b, c)); +} // No FMA instruction for int, so use MLA unconditionally. template <> @@ -5242,13 +5274,28 @@ EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vfmaq_f64(c, a, b); } +template <> +EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { + return vfmsq_f64(c, a, b); +} #else template <> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vmlaq_f64(c, a, b); } +template <> +EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { + return vmlsq_f64(c, a, b); +} #endif - +template <> +EIGEN_STRONG_INLINE Packet2d pmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) { + return pnegate(pnmadd(a, b, c)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pnmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) { + return pnegate(pmadd(a, b, c)); +} template <> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return vminq_f64(a, b); @@ -5657,18 +5704,33 @@ } template <> -EIGEN_STRONG_INLINE Packet8hf pmsub(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) { - return vfmaq_f16(pnegate(c), a, b); +EIGEN_STRONG_INLINE Packet8hf pnmadd(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) { + return vfmsq_f16(c, a, b); } template <> EIGEN_STRONG_INLINE Packet4hf pnmadd(const Packet4hf& a, const Packet4hf& b, const Packet4hf& c) { - return vfma_f16(c, pnegate(a), b); + return vfms_f16(c, a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8hf pmsub(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) { + return pnegate(pnmadd(a, b, c)); +} + +template <> +EIGEN_STRONG_INLINE Packet4hf pmsub(const Packet4hf& a, const Packet4hf& b, const Packet4hf& c) { + return pnegate(pnmadd(a, b, c)); +} + +template <> +EIGEN_STRONG_INLINE Packet8hf pnmsub(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) { + return pnegate(pmadd(a, b, c)); } template <> EIGEN_STRONG_INLINE Packet4hf pnmsub(const Packet4hf& a, const Packet4hf& b, const Packet4hf& c) { - return vfma_f16(pnegate(c), pnegate(a), b); + return pnegate(pmadd(a, b, c)); } template <>
diff --git a/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h b/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h index 56743da..913beb6 100644 --- a/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h +++ b/Eigen/src/Core/products/GeneralMatrixMatrix_BLAS.h
@@ -55,7 +55,7 @@ ConjugateRhs, ColMajor, 1> { \ typedef gebp_traits<EIGTYPE, EIGTYPE> Traits; \ \ - static void run(Index rows, Index cols, Index depth, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \ + static void run(Index rows, Index cols, Index depth, const EIGTYPE* lhs_, Index lhsStride, const EIGTYPE* rhs_, \ Index rhsStride, EIGTYPE* res, Index resIncr, Index resStride, EIGTYPE alpha, \ level3_blocking<EIGTYPE, EIGTYPE>& /*blocking*/, GemmParallelInfo<Index>* /*info = 0*/) { \ using std::conj; \ @@ -84,20 +84,20 @@ \ /* Set a, b, c */ \ if ((LhsStorageOrder == ColMajor) && (ConjugateLhs)) { \ - Map<const MatrixX##EIGPREFIX, 0, OuterStride<> > lhs(_lhs, m, k, OuterStride<>(lhsStride)); \ + Map<const MatrixX##EIGPREFIX, 0, OuterStride<> > lhs(lhs_, m, k, OuterStride<>(lhsStride)); \ a_tmp = lhs.conjugate(); \ a = a_tmp.data(); \ lda = convert_index<BlasIndex>(a_tmp.outerStride()); \ } else \ - a = _lhs; \ + a = lhs_; \ \ if ((RhsStorageOrder == ColMajor) && (ConjugateRhs)) { \ - Map<const MatrixX##EIGPREFIX, 0, OuterStride<> > rhs(_rhs, k, n, OuterStride<>(rhsStride)); \ + Map<const MatrixX##EIGPREFIX, 0, OuterStride<> > rhs(rhs_, k, n, OuterStride<>(rhsStride)); \ b_tmp = rhs.conjugate(); \ b = b_tmp.data(); \ ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \ } else \ - b = _rhs; \ + b = rhs_; \ \ BLASFUNC(&transa, &transb, &m, &n, &k, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \ (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \ @@ -116,6 +116,88 @@ GEMM_SPECIALIZATION(scomplex, cf, float, cgemm_) #endif +// If OpenBLAS with BUILD_BFLOAT16=1 support is available, +// use sbgemm for bfloat16. +#if EIGEN_USE_OPENBLAS_BFLOAT16 + +extern "C" { +// OpenBLAS prototype. +void sbgemm_(const char* trans_a, const char* trans_b, const int* M, const int* N, const int* K, const float* alpha, + const Eigen::bfloat16* A, const int* lda, const Eigen::bfloat16* B, const int* ldb, const float* beta, + float* C, const int* ldc); +} // extern "C" + +template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> +struct general_matrix_matrix_product<Index, Eigen::bfloat16, LhsStorageOrder, ConjugateLhs, Eigen::bfloat16, + RhsStorageOrder, ConjugateRhs, ColMajor, 1> { + typedef gebp_traits<Eigen::bfloat16, Eigen::bfloat16> Traits; + + static void run(Index rows, Index cols, Index depth, const Eigen::bfloat16* lhs_, Index lhsStride, + const Eigen::bfloat16* rhs_, Index rhsStride, Eigen::bfloat16* res, Index resIncr, Index resStride, + Eigen::bfloat16 alpha, level3_blocking<Eigen::bfloat16, Eigen::bfloat16>& /*blocking*/, + GemmParallelInfo<Index>* /*info = 0*/) { + using std::conj; + if (rows == 0 || cols == 0 || depth == 0) return; + EIGEN_ONLY_USED_FOR_DEBUG(resIncr); + eigen_assert(resIncr == 1); + char transa, transb; + BlasIndex m, n, k, lda, ldb, ldc; + const Eigen::bfloat16 *a, *b; + + float falpha = static_cast<float>(alpha); + float fbeta = float(1.0); + + using MatrixXbf16 = Matrix<Eigen::bfloat16, Dynamic, Dynamic>; + MatrixXbf16 a_tmp, b_tmp; + MatrixXf r_tmp; + + /* Set transpose options */ + transa = (LhsStorageOrder == RowMajor) ? ((ConjugateLhs) ? 'C' : 'T') : 'N'; + transb = (RhsStorageOrder == RowMajor) ? ((ConjugateRhs) ? 'C' : 'T') : 'N'; + + /* Set m, n, k */ + m = convert_index<BlasIndex>(rows); + n = convert_index<BlasIndex>(cols); + k = convert_index<BlasIndex>(depth); + + /* Set lda, ldb, ldc */ + lda = convert_index<BlasIndex>(lhsStride); + ldb = convert_index<BlasIndex>(rhsStride); + ldc = convert_index<BlasIndex>(m); + + /* Set a, b, c */ + if ((LhsStorageOrder == ColMajor) && (ConjugateLhs)) { + Map<const MatrixXbf16, 0, OuterStride<> > lhs(lhs_, m, k, OuterStride<>(lhsStride)); + a_tmp = lhs.conjugate(); + a = a_tmp.data(); + lda = convert_index<BlasIndex>(a_tmp.outerStride()); + } else { + a = lhs_; + } + + if ((RhsStorageOrder == ColMajor) && (ConjugateRhs)) { + Map<const MatrixXbf16, 0, OuterStride<> > rhs(rhs_, k, n, OuterStride<>(rhsStride)); + b_tmp = rhs.conjugate(); + b = b_tmp.data(); + ldb = convert_index<BlasIndex>(b_tmp.outerStride()); + } else { + b = rhs_; + } + + // Evaluate to a temporary intermediate array. + r_tmp.resize(m, n); + + sbgemm_(&transa, &transb, &m, &n, &k, (const float*)&numext::real_ref(falpha), a, &lda, b, &ldb, + (const float*)&numext::real_ref(fbeta), r_tmp.data(), &ldc); + + // Cast to the output. + Map<MatrixXbf16, 0, OuterStride<> > result(res, m, n, OuterStride<>(resStride)); + result = r_tmp.cast<Eigen::bfloat16>(); + } +}; + +#endif // EIGEN_USE_OPENBLAS_SBGEMM + } // namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/util/Memory.h b/Eigen/src/Core/util/Memory.h index 89b2fff..44056b3 100644 --- a/Eigen/src/Core/util/Memory.h +++ b/Eigen/src/Core/util/Memory.h
@@ -762,7 +762,7 @@ * This is accomplished through alloca if this later is supported and if the required number of bytes * is below EIGEN_STACK_ALLOCATION_LIMIT. */ -#ifdef EIGEN_ALLOCA +#if defined(EIGEN_ALLOCA) && !defined(EIGEN_NO_ALLOCA) #if EIGEN_DEFAULT_ALIGN_BYTES > 0 // We always manually re-align the result of EIGEN_ALLOCA. @@ -785,14 +785,14 @@ #define EIGEN_ALIGNED_ALLOCA(SIZE) EIGEN_ALLOCA(SIZE) #endif -#define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \ - Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \ - TYPE* NAME = (BUFFER) != 0 ? (BUFFER) \ - : reinterpret_cast<TYPE*>((sizeof(TYPE) * SIZE <= EIGEN_STACK_ALLOCATION_LIMIT) \ - ? EIGEN_ALIGNED_ALLOCA(sizeof(TYPE) * SIZE) \ - : Eigen::internal::aligned_malloc(sizeof(TYPE) * SIZE)); \ - Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \ - (BUFFER) == 0 ? NAME : 0, SIZE, sizeof(TYPE) * SIZE > EIGEN_STACK_ALLOCATION_LIMIT) +#define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \ + Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \ + TYPE* NAME = (BUFFER) != 0 ? (BUFFER) \ + : reinterpret_cast<TYPE*>((sizeof(TYPE) * (SIZE) <= EIGEN_STACK_ALLOCATION_LIMIT) \ + ? EIGEN_ALIGNED_ALLOCA(sizeof(TYPE) * (SIZE)) \ + : Eigen::internal::aligned_malloc(sizeof(TYPE) * (SIZE))); \ + Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \ + (BUFFER) == 0 ? NAME : 0, SIZE, sizeof(TYPE) * (SIZE) > EIGEN_STACK_ALLOCATION_LIMIT) #define ei_declare_local_nested_eval(XPR_T, XPR, N, NAME) \ Eigen::internal::local_nested_eval_wrapper<XPR_T, N> EIGEN_CAT(NAME, _wrapper)( \ @@ -805,10 +805,11 @@ #else -#define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \ - Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \ - TYPE* NAME = (BUFFER) != 0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::aligned_malloc(sizeof(TYPE) * SIZE)); \ - Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \ +#define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \ + Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \ + TYPE* NAME = \ + (BUFFER) != 0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::aligned_malloc(sizeof(TYPE) * (SIZE))); \ + Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \ (BUFFER) == 0 ? NAME : 0, SIZE, true) #define ei_declare_local_nested_eval(XPR_T, XPR, N, NAME) \
diff --git a/test/half_float.cpp b/test/half_float.cpp index 90ac825..aabe896 100644 --- a/test/half_float.cpp +++ b/test/half_float.cpp
@@ -72,17 +72,16 @@ // NaNs and infinities. VERIFY(!(numext::isinf)(float(half(65504.0f)))); // Largest finite number. VERIFY(!(numext::isnan)(float(half(0.0f)))); + VERIFY((numext::isfinite)(float(half(65504.0f)))); + VERIFY((numext::isfinite)(float(half(0.0f)))); VERIFY((numext::isinf)(float(half(__half_raw(0xfc00))))); VERIFY((numext::isnan)(float(half(__half_raw(0xfc01))))); VERIFY((numext::isinf)(float(half(__half_raw(0x7c00))))); VERIFY((numext::isnan)(float(half(__half_raw(0x7c01))))); -#if !EIGEN_COMP_MSVC - // Visual Studio errors out on divisions by 0 - VERIFY((numext::isnan)(float(half(0.0 / 0.0)))); - VERIFY((numext::isinf)(float(half(1.0 / 0.0)))); - VERIFY((numext::isinf)(float(half(-1.0 / 0.0)))); -#endif + VERIFY((numext::isnan)(float(NumTraits<half>::quiet_NaN()))); + VERIFY((numext::isinf)(float(NumTraits<half>::infinity()))); + VERIFY((numext::isinf)(float(-NumTraits<half>::infinity()))); // Exactly same checks as above, just directly on the half representation. VERIFY(!(numext::isinf)(half(__half_raw(0x7bff)))); @@ -92,12 +91,9 @@ VERIFY((numext::isinf)(half(__half_raw(0x7c00)))); VERIFY((numext::isnan)(half(__half_raw(0x7c01)))); -#if !EIGEN_COMP_MSVC - // Visual Studio errors out on divisions by 0 - VERIFY((numext::isnan)(half(0.0 / 0.0))); - VERIFY((numext::isinf)(half(1.0 / 0.0))); - VERIFY((numext::isinf)(half(-1.0 / 0.0))); -#endif + VERIFY((numext::isnan)(NumTraits<half>::quiet_NaN())); + VERIFY((numext::isinf)(NumTraits<half>::infinity())); + VERIFY((numext::isinf)(-NumTraits<half>::infinity())); // Conversion to bool VERIFY(!static_cast<bool>(half(0.0))); @@ -204,19 +200,25 @@ VERIFY(half(1.0f) != half(2.0f)); // Comparisons with NaNs and infinities. -#if !EIGEN_COMP_MSVC - // Visual Studio errors out on divisions by 0 - VERIFY(!(half(0.0 / 0.0) == half(0.0 / 0.0))); - VERIFY(half(0.0 / 0.0) != half(0.0 / 0.0)); + VERIFY(!(NumTraits<half>::quiet_NaN() == NumTraits<half>::quiet_NaN())); + VERIFY(NumTraits<half>::quiet_NaN() != NumTraits<half>::quiet_NaN()); - VERIFY(!(half(1.0) == half(0.0 / 0.0))); - VERIFY(!(half(1.0) < half(0.0 / 0.0))); - VERIFY(!(half(1.0) > half(0.0 / 0.0))); - VERIFY(half(1.0) != half(0.0 / 0.0)); + VERIFY(!(internal::random<half>() == NumTraits<half>::quiet_NaN())); + VERIFY(!(internal::random<half>() < NumTraits<half>::quiet_NaN())); + VERIFY(!(internal::random<half>() > NumTraits<half>::quiet_NaN())); + VERIFY(!(internal::random<half>() <= NumTraits<half>::quiet_NaN())); + VERIFY(!(internal::random<half>() >= NumTraits<half>::quiet_NaN())); + VERIFY(internal::random<half>() != NumTraits<half>::quiet_NaN()); - VERIFY(half(1.0) < half(1.0 / 0.0)); - VERIFY(half(1.0) > half(-1.0 / 0.0)); -#endif + VERIFY(!(NumTraits<half>::quiet_NaN() == internal::random<half>())); + VERIFY(!(NumTraits<half>::quiet_NaN() < internal::random<half>())); + VERIFY(!(NumTraits<half>::quiet_NaN() > internal::random<half>())); + VERIFY(!(NumTraits<half>::quiet_NaN() <= internal::random<half>())); + VERIFY(!(NumTraits<half>::quiet_NaN() >= internal::random<half>())); + VERIFY(NumTraits<half>::quiet_NaN() != internal::random<half>()); + + VERIFY(internal::random<half>() < NumTraits<half>::infinity()); + VERIFY(internal::random<half>() > -NumTraits<half>::infinity()); } void test_basic_functions() {