Update Eigen to commit:9b511fe4fecf93c3438c43440c760dfa6f723b43 CHANGELOG ========= 9b511fe4f - Fix cxx11_tensor_expr.cpp 943fdc71c - Use more FMA in reciprocal iteration for precision 1133aa82c - fix various compiler warnings 8ae3b1aaa - Fix loongarch unsigned pabsdiff 035cf6849 - Fix build of realview.cpp 23a5482fc - Misc. packet math cleanups. 4cb0776f8 - Add 5.0.1 release notes and a few unreleased features. 8b85f5933 - Fix realview ffcd7bdbd - Avoid breaking the build on older compilers. da867c31c - Fix defines in AVX512 custom TRSM kernel. PiperOrigin-RevId: 830969130 Change-Id: Ifcafaaeed12734373fa9cc4742ce48267e7063c4
diff --git a/Eigen/src/Core/DenseBase.h b/Eigen/src/Core/DenseBase.h index c81e1d1..dcadcbf 100644 --- a/Eigen/src/Core/DenseBase.h +++ b/Eigen/src/Core/DenseBase.h
@@ -367,11 +367,11 @@ EIGEN_DEVICE_FUNC inline bool allFinite() const; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator*=(const Scalar& other); - template <bool Enable = !internal::is_same<Scalar, RealScalar>::value, typename = std::enable_if_t<Enable>> + template <bool Enable = internal::complex_array_access<Scalar>::value, typename = std::enable_if_t<Enable>> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator*=(const RealScalar& other); EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator/=(const Scalar& other); - template <bool Enable = !internal::is_same<Scalar, RealScalar>::value, typename = std::enable_if_t<Enable>> + template <bool Enable = internal::complex_array_access<Scalar>::value, typename = std::enable_if_t<Enable>> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator/=(const RealScalar& other); typedef internal::add_const_on_value_type_t<typename internal::eval<Derived>::type> EvalReturnType;
diff --git a/Eigen/src/Core/Dot.h b/Eigen/src/Core/Dot.h index 059527c..a173306 100644 --- a/Eigen/src/Core/Dot.h +++ b/Eigen/src/Core/Dot.h
@@ -20,10 +20,7 @@ template <typename Derived, typename Scalar = typename traits<Derived>::Scalar> struct squared_norm_impl { using Real = typename NumTraits<Scalar>::Real; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Real run(const Derived& a) { - Scalar result = a.unaryExpr(squared_norm_functor<Scalar>()).sum(); - return numext::real(result) + numext::imag(result); - } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Real run(const Derived& a) { return a.realView().cwiseAbs2().sum(); } }; template <typename Derived>
diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index bfb52ab..ba38cd1 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h
@@ -57,12 +57,12 @@ HasConj = 1, HasSetLinear = 1, HasSign = 1, + HasAbsDiff = 1, // By default, the nearest integer functions (rint, round, floor, ceil, trunc) are enabled for all scalar and packet // types HasRound = 1, HasArg = 0, - HasAbsDiff = 0, // This flag is used to indicate whether packet comparison is supported. // pcmp_eq and pcmp_lt should be defined for it to be true. HasCmp = 0, @@ -116,6 +116,7 @@ enum { HasAdd = 0, HasSub = 0, + HasAbsDiff = 0, HasMul = 0, HasNegate = 0, HasAbs = 0, @@ -130,17 +131,18 @@ template <typename T> struct packet_traits<const T> : packet_traits<T> {}; +struct default_unpacket_traits { + enum { vectorizable = false, masked_load_available = false, masked_store_available = false }; +}; + template <typename T> -struct unpacket_traits { +struct unpacket_traits : default_unpacket_traits { typedef T type; typedef T half; typedef typename numext::get_integer_by_size<sizeof(T)>::signed_type integer_packet; enum { size = 1, alignment = alignof(T), - vectorizable = false, - masked_load_available = false, - masked_store_available = false }; }; @@ -747,9 +749,15 @@ /** \internal \returns the min of \a a and \a b (coeff-wise) */ template <typename Packet> -EIGEN_DEVICE_FUNC inline Packet pabsdiff(const Packet& a, const Packet& b) { +EIGEN_DEVICE_FUNC inline std::enable_if_t<NumTraits<typename unpacket_traits<Packet>::type>::IsInteger, Packet> +pabsdiff(const Packet& a, const Packet& b) { return pselect(pcmp_lt(a, b), psub(b, a), psub(a, b)); } +template <typename Packet> +EIGEN_DEVICE_FUNC inline std::enable_if_t<!NumTraits<typename unpacket_traits<Packet>::type>::IsInteger, Packet> +pabsdiff(const Packet& a, const Packet& b) { + return pabs(psub(a, b)); +} /** \internal \returns a packet version of \a *from, from must be properly aligned */ template <typename Packet>
diff --git a/Eigen/src/Core/MathFunctionsImpl.h b/Eigen/src/Core/MathFunctionsImpl.h index c4b5da3..43d9d64 100644 --- a/Eigen/src/Core/MathFunctionsImpl.h +++ b/Eigen/src/Core/MathFunctionsImpl.h
@@ -37,15 +37,16 @@ static_assert(Steps > 0, "Steps must be at least 1."); EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet run(const Packet& a, const Packet& approx_a_recip) { using Scalar = typename unpacket_traits<Packet>::type; - const Packet two = pset1<Packet>(Scalar(2)); + const Packet one = pset1<Packet>(Scalar(1)); // Refine the approximation using one Newton-Raphson step: // x_{i} = x_{i-1} * (2 - a * x_{i-1}) const Packet x = generic_reciprocal_newton_step<Packet, Steps - 1>::run(a, approx_a_recip); - const Packet tmp = pnmadd(a, x, two); + const Packet tmp = pnmadd(a, x, one); // If tmp is NaN, it means that a is either +/-0 or +/-Inf. // In this case return the approximation directly. const Packet is_not_nan = pcmp_eq(tmp, tmp); - return pselect(is_not_nan, pmul(x, tmp), x); + // Use two FMAs instead of FMA+FMUL to improve precision. + return pselect(is_not_nan, pmadd(x, tmp, x), x); } };
diff --git a/Eigen/src/Core/RealView.h b/Eigen/src/Core/RealView.h index 7ba42f9..3be5556 100644 --- a/Eigen/src/Core/RealView.h +++ b/Eigen/src/Core/RealView.h
@@ -17,20 +17,16 @@ namespace internal { -// Vectorized assignment to RealView requires array-oriented access to the real and imaginary components. +// Write access and vectorization requires array-oriented access to the real and imaginary components. // From https://en.cppreference.com/w/cpp/numeric/complex.html: // For any pointer to an element of an array of std::complex<T> named p and any valid array index i, // reinterpret_cast<T*>(p)[2 * i] is the real part of the complex number p[i], and // reinterpret_cast<T*>(p)[2 * i + 1] is the imaginary part of the complex number p[i]. -template <typename ComplexScalar> +template <typename T> struct complex_array_access : std::false_type {}; -template <> -struct complex_array_access<std::complex<float>> : std::true_type {}; -template <> -struct complex_array_access<std::complex<double>> : std::true_type {}; -template <> -struct complex_array_access<std::complex<long double>> : std::true_type {}; +template <typename T> +struct complex_array_access<std::complex<T>> : std::true_type {}; template <typename Xpr> struct traits<RealView<Xpr>> : public traits<Xpr> { @@ -40,13 +36,17 @@ if (size_as_int == Dynamic) return Dynamic; return times_two ? (2 * size_as_int) : size_as_int; } + using Base = traits<Xpr>; using ComplexScalar = typename Base::Scalar; using Scalar = typename NumTraits<ComplexScalar>::Real; - static constexpr int ActualDirectAccessBit = complex_array_access<ComplexScalar>::value ? DirectAccessBit : 0; + + static constexpr bool ArrayAccess = complex_array_access<ComplexScalar>::value; + static constexpr int ActualDirectAccessBit = ArrayAccess ? DirectAccessBit : 0; + static constexpr int ActualLvaluebit = !std::is_const<Xpr>::value && ArrayAccess ? LvalueBit : 0; static constexpr int ActualPacketAccessBit = packet_traits<Scalar>::Vectorizable ? PacketAccessBit : 0; static constexpr int FlagMask = - ActualDirectAccessBit | ActualPacketAccessBit | HereditaryBits | LinearAccessBit | LvalueBit; + ActualDirectAccessBit | ActualLvaluebit | ActualPacketAccessBit | HereditaryBits | LinearAccessBit; static constexpr int BaseFlags = int(evaluator<Xpr>::Flags) | int(Base::Flags); static constexpr int Flags = BaseFlags & FlagMask; static constexpr bool IsRowMajor = Flags & RowMajorBit; @@ -66,68 +66,84 @@ using XprType = RealView<Xpr>; using ExpressionTraits = traits<XprType>; using ComplexScalar = typename ExpressionTraits::ComplexScalar; - using ComplexCoeffReturnType = typename BaseEvaluator::CoeffReturnType; using Scalar = typename ExpressionTraits::Scalar; - static constexpr bool IsRowMajor = ExpressionTraits::IsRowMajor; static constexpr int Flags = ExpressionTraits::Flags; static constexpr int CoeffReadCost = BaseEvaluator::CoeffReadCost; static constexpr int Alignment = BaseEvaluator::Alignment; + static constexpr bool IsRowMajor = ExpressionTraits::IsRowMajor; + static constexpr bool DirectAccess = Flags & DirectAccessBit; + + using ComplexCoeffReturnType = std::conditional_t<DirectAccess, const ComplexScalar&, ComplexScalar>; + using CoeffReturnType = std::conditional_t<DirectAccess, const Scalar&, Scalar>; EIGEN_DEVICE_FUNC explicit evaluator(XprType realView) : BaseEvaluator(realView.m_xpr) {} - template <bool Enable = std::is_reference<ComplexCoeffReturnType>::value, typename = std::enable_if_t<!Enable>> + template <bool Enable = DirectAccess, std::enable_if_t<!Enable, bool> = true> constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index row, Index col) const { - ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(IsRowMajor ? row : row / 2, IsRowMajor ? col / 2 : col); - Index p = (IsRowMajor ? col : row) & 1; - return p ? numext::real(cscalar) : numext::imag(cscalar); + Index r = IsRowMajor ? row : row / 2; + Index c = IsRowMajor ? col / 2 : col; + bool p = (IsRowMajor ? col : row) & 1; + ComplexScalar ccoeff = BaseEvaluator::coeff(r, c); + return p ? numext::imag(ccoeff) : numext::real(ccoeff); } - - template <bool Enable = std::is_reference<ComplexCoeffReturnType>::value, typename = std::enable_if_t<Enable>> - constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index row, Index col) const { - ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(IsRowMajor ? row : row / 2, IsRowMajor ? col / 2 : col); + template <bool Enable = DirectAccess, std::enable_if_t<Enable, bool> = true> + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { + Index r = IsRowMajor ? row : row / 2; + Index c = IsRowMajor ? col / 2 : col; Index p = (IsRowMajor ? col : row) & 1; - return reinterpret_cast<const Scalar(&)[2]>(cscalar)[p]; + ComplexCoeffReturnType ccoeff = BaseEvaluator::coeff(r, c); + return reinterpret_cast<const Scalar(&)[2]>(ccoeff)[p]; } - - constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { - ComplexScalar& cscalar = BaseEvaluator::coeffRef(IsRowMajor ? row : row / 2, IsRowMajor ? col / 2 : col); - Index p = (IsRowMajor ? col : row) & 1; - return reinterpret_cast<Scalar(&)[2]>(cscalar)[p]; - } - - template <bool Enable = std::is_reference<ComplexCoeffReturnType>::value, typename = std::enable_if_t<!Enable>> + template <bool Enable = DirectAccess, std::enable_if_t<!Enable, bool> = true> constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index index) const { - ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(index / 2); - Index p = index & 1; - return p ? numext::real(cscalar) : numext::imag(cscalar); + ComplexScalar ccoeff = BaseEvaluator::coeff(index / 2); + bool p = index & 1; + return p ? numext::imag(ccoeff) : numext::real(ccoeff); } - - template <bool Enable = std::is_reference<ComplexCoeffReturnType>::value, typename = std::enable_if_t<Enable>> - constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index index) const { - ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(index / 2); + template <bool Enable = DirectAccess, std::enable_if_t<Enable, bool> = true> + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { + ComplexCoeffReturnType ccoeff = BaseEvaluator::coeff(index / 2); Index p = index & 1; - return reinterpret_cast<const Scalar(&)[2]>(cscalar)[p]; + return reinterpret_cast<const Scalar(&)[2]>(ccoeff)[p]; } - + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { + Index r = IsRowMajor ? row : row / 2; + Index c = IsRowMajor ? col / 2 : col; + Index p = (IsRowMajor ? col : row) & 1; + ComplexScalar& ccoeffRef = BaseEvaluator::coeffRef(r, c); + return reinterpret_cast<Scalar(&)[2]>(ccoeffRef)[p]; + } constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { - ComplexScalar& cscalar = BaseEvaluator::coeffRef(index / 2); + ComplexScalar& ccoeffRef = BaseEvaluator::coeffRef(index / 2); Index p = index & 1; - return reinterpret_cast<Scalar(&)[2]>(cscalar)[p]; + return reinterpret_cast<Scalar(&)[2]>(ccoeffRef)[p]; } + // If the first index is odd (imaginary), discard the first scalar + // in 'result' and assign the missing scalar. + // This operation is safe as the real component of the first scalar must exist. + template <int LoadMode, typename PacketType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { constexpr int RealPacketSize = unpacket_traits<PacketType>::size; using ComplexPacket = typename find_packet_by_size<ComplexScalar, RealPacketSize / 2>::type; EIGEN_STATIC_ASSERT((find_packet_by_size<ComplexScalar, RealPacketSize / 2>::value), MISSING COMPATIBLE COMPLEX PACKET TYPE) - eigen_assert(((IsRowMajor ? col : row) % 2 == 0) && "the inner index must be even"); - - Index crow = IsRowMajor ? row : row / 2; - Index ccol = IsRowMajor ? col / 2 : col; - ComplexPacket cpacket = BaseEvaluator::template packet<LoadMode, ComplexPacket>(crow, ccol); - return preinterpret<PacketType, ComplexPacket>(cpacket); + Index r = IsRowMajor ? row : row / 2; + Index c = IsRowMajor ? col / 2 : col; + bool p = (IsRowMajor ? col : row) & 1; + ComplexPacket cresult = BaseEvaluator::template packet<LoadMode, ComplexPacket>(r, c); + PacketType result = preinterpret<PacketType>(cresult); + if (p) { + Scalar aux[RealPacketSize + 1]; + pstoreu(aux, result); + Index lastr = IsRowMajor ? row : row + RealPacketSize - 1; + Index lastc = IsRowMajor ? col + RealPacketSize - 1 : col; + aux[RealPacketSize] = coeff(lastr, lastc); + result = ploadu<PacketType>(aux + 1); + } + return result; } template <int LoadMode, typename PacketType> @@ -136,28 +152,48 @@ using ComplexPacket = typename find_packet_by_size<ComplexScalar, RealPacketSize / 2>::type; EIGEN_STATIC_ASSERT((find_packet_by_size<ComplexScalar, RealPacketSize / 2>::value), MISSING COMPATIBLE COMPLEX PACKET TYPE) - eigen_assert((index % 2 == 0) && "the index must be even"); - - Index cindex = index / 2; - ComplexPacket cpacket = BaseEvaluator::template packet<LoadMode, ComplexPacket>(cindex); - return preinterpret<PacketType, ComplexPacket>(cpacket); + ComplexPacket cresult = BaseEvaluator::template packet<LoadMode, ComplexPacket>(index / 2); + PacketType result = preinterpret<PacketType>(cresult); + bool p = index & 1; + if (p) { + Scalar aux[RealPacketSize + 1]; + pstoreu(aux, result); + aux[RealPacketSize] = coeff(index + RealPacketSize - 1); + result = ploadu<PacketType>(aux + 1); + } + return result; } + // The requested real packet segment forms the half-open interval [begin, end), where 'end' = 'begin' + 'count'. + // In order to access the underlying complex array, even indices must be aligned with the real components + // of the complex scalars. 'begin' and 'count' must be modified as follows: + // a) 'begin' must be rounded down to the nearest even number; and + // b) 'end' must be rounded up to the nearest even number. + template <int LoadMode, typename PacketType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index row, Index col, Index begin, Index count) const { constexpr int RealPacketSize = unpacket_traits<PacketType>::size; using ComplexPacket = typename find_packet_by_size<ComplexScalar, RealPacketSize / 2>::type; EIGEN_STATIC_ASSERT((find_packet_by_size<ComplexScalar, RealPacketSize / 2>::value), MISSING COMPATIBLE COMPLEX PACKET TYPE) - eigen_assert(((IsRowMajor ? col : row) % 2 == 0) && "the inner index must be even"); - eigen_assert((begin % 2 == 0) && (count % 2 == 0) && "begin and count must be even"); - - Index crow = IsRowMajor ? row : row / 2; - Index ccol = IsRowMajor ? col / 2 : col; - Index cbegin = begin / 2; - Index ccount = count / 2; - ComplexPacket cpacket = BaseEvaluator::template packetSegment<LoadMode, ComplexPacket>(crow, ccol, cbegin, ccount); - return preinterpret<PacketType, ComplexPacket>(cpacket); + Index actualBegin = numext::round_down(begin, 2); + Index actualEnd = numext::round_down(begin + count + 1, 2); + Index actualCount = actualEnd - actualBegin; + Index r = IsRowMajor ? row : row / 2; + Index c = IsRowMajor ? col / 2 : col; + ComplexPacket cresult = + BaseEvaluator::template packetSegment<LoadMode, ComplexPacket>(r, c, actualBegin / 2, actualCount / 2); + PacketType result = preinterpret<PacketType>(cresult); + bool p = (IsRowMajor ? col : row) & 1; + if (p) { + Scalar aux[RealPacketSize + 1] = {}; + pstoreu(aux, result); + Index lastr = IsRowMajor ? row : row + actualEnd - 1; + Index lastc = IsRowMajor ? col + actualEnd - 1 : col; + aux[actualEnd] = coeff(lastr, lastc); + result = ploadu<PacketType>(aux + 1); + } + return result; } template <int LoadMode, typename PacketType> @@ -166,14 +202,20 @@ using ComplexPacket = typename find_packet_by_size<ComplexScalar, RealPacketSize / 2>::type; EIGEN_STATIC_ASSERT((find_packet_by_size<ComplexScalar, RealPacketSize / 2>::value), MISSING COMPATIBLE COMPLEX PACKET TYPE) - eigen_assert((index % 2 == 0) && "the index must be even"); - eigen_assert((begin % 2 == 0) && (count % 2 == 0) && "begin and count must be even"); - - Index cindex = index / 2; - Index cbegin = begin / 2; - Index ccount = count / 2; - ComplexPacket cpacket = BaseEvaluator::template packetSegment<LoadMode, ComplexPacket>(cindex, cbegin, ccount); - return preinterpret<PacketType, ComplexPacket>(cpacket); + Index actualBegin = numext::round_down(begin, 2); + Index actualEnd = numext::round_down(begin + count + 1, 2); + Index actualCount = actualEnd - actualBegin; + ComplexPacket cresult = + BaseEvaluator::template packetSegment<LoadMode, ComplexPacket>(index / 2, actualBegin / 2, actualCount / 2); + PacketType result = preinterpret<PacketType>(cresult); + bool p = index & 1; + if (p) { + Scalar aux[RealPacketSize + 1] = {}; + pstoreu(aux, result); + aux[actualEnd] = coeff(index + actualEnd - 1); + result = ploadu<PacketType>(aux + 1); + } + return result; } }; @@ -211,7 +253,7 @@ EIGEN_DEVICE_FUNC RealView& operator=(const DenseBase<OtherDerived>& other); protected: - friend struct internal::evaluator<RealView<Xpr>>; + friend struct internal::evaluator<RealView>; Xpr& m_xpr; };
diff --git a/Eigen/src/Core/arch/AVX/PacketMath.h b/Eigen/src/Core/arch/AVX/PacketMath.h index 48fe815..318b375 100644 --- a/Eigen/src/Core/arch/AVX/PacketMath.h +++ b/Eigen/src/Core/arch/AVX/PacketMath.h
@@ -250,7 +250,6 @@ HasDiv = 0, HasNegate = 0, - HasSqrt = 0, HasCmp = 1, HasMin = 1, @@ -277,12 +276,9 @@ AlignedOnScalar = 1, size = 4, - // HasMin = 0, - // HasMax = 0, HasDiv = 0, HasTranspose = 0, HasNegate = 0, - HasSqrt = 0, HasCmp = 1, HasShift = 1 };
diff --git a/Eigen/src/Core/arch/AVX512/GemmKernel.h b/Eigen/src/Core/arch/AVX512/GemmKernel.h index 483e149..96157f4 100644 --- a/Eigen/src/Core/arch/AVX512/GemmKernel.h +++ b/Eigen/src/Core/arch/AVX512/GemmKernel.h
@@ -81,10 +81,10 @@ Index m; const Index n, k, ldc; const Index inc; - const Scalar *alpha; + const Scalar* alpha; const Scalar *a, *b; - Scalar *c; + Scalar* c; const bool is_alpha1; const bool is_beta0; @@ -92,26 +92,26 @@ const Index a_stride, b_stride; const Index a_off, b_off; - EIGEN_ALWAYS_INLINE void prefetch_a(const Scalar *a_addr) { - _mm_prefetch((char *)(a_prefetch_size + a_addr - a_shift), _MM_HINT_T0); + EIGEN_ALWAYS_INLINE void prefetch_a(const Scalar* a_addr) { + _mm_prefetch((char*)(a_prefetch_size + a_addr - a_shift), _MM_HINT_T0); } - EIGEN_ALWAYS_INLINE void prefetch_b(const Scalar *b_addr) { - _mm_prefetch((char *)(b_prefetch_size + b_addr - b_shift), _MM_HINT_T0); + EIGEN_ALWAYS_INLINE void prefetch_b(const Scalar* b_addr) { + _mm_prefetch((char*)(b_prefetch_size + b_addr - b_shift), _MM_HINT_T0); } - EIGEN_ALWAYS_INLINE void prefetch_x(const Scalar *x_addr) { _mm_prefetch((char *)(x_addr - a_shift), _MM_HINT_T2); } + EIGEN_ALWAYS_INLINE void prefetch_x(const Scalar* x_addr) { _mm_prefetch((char*)(x_addr - a_shift), _MM_HINT_T2); } - EIGEN_ALWAYS_INLINE void prefetch_c(const Scalar *c_addr) { + EIGEN_ALWAYS_INLINE void prefetch_c(const Scalar* c_addr) { #if defined(__PRFCHW__) && __PRFCHW__ == 1 - _m_prefetchw((void *)c_addr); + _m_prefetchw((void*)c_addr); #else - _mm_prefetch((char *)c_addr, _MM_HINT_T0); + _mm_prefetch((char*)c_addr, _MM_HINT_T0); #endif } template <int nelems> - EIGEN_ALWAYS_INLINE void a_load(vec &a_reg, const Scalar *a_addr) { + EIGEN_ALWAYS_INLINE void a_load(vec& a_reg, const Scalar* a_addr) { switch (nelems * sizeof(*a_addr) * 8) { default: case 512 * 3: @@ -124,13 +124,13 @@ a_reg = ploadu<vec>(a_addr); break; case 256 * 1: - a_reg = preinterpret<vec>(_mm512_broadcast_f64x4(ploadu<Packet4d>(reinterpret_cast<const double *>(a_addr)))); + a_reg = preinterpret<vec>(_mm512_broadcast_f64x4(ploadu<Packet4d>(reinterpret_cast<const double*>(a_addr)))); break; case 128 * 1: - a_reg = preinterpret<vec>(_mm512_broadcast_f32x4(ploadu<Packet4f>(reinterpret_cast<const float *>(a_addr)))); + a_reg = preinterpret<vec>(_mm512_broadcast_f32x4(ploadu<Packet4f>(reinterpret_cast<const float*>(a_addr)))); break; case 64 * 1: - a_reg = preinterpret<vec>(pload1<Packet8d>(reinterpret_cast<const double *>(a_addr))); + a_reg = preinterpret<vec>(pload1<Packet8d>(reinterpret_cast<const double*>(a_addr))); break; case 32 * 1: a_reg = pload1<vec>(a_addr); @@ -138,10 +138,10 @@ } } - EIGEN_ALWAYS_INLINE void b_load(vec &b_reg, const Scalar *b_addr) { b_reg = pload1<vec>(b_addr); } + EIGEN_ALWAYS_INLINE void b_load(vec& b_reg, const Scalar* b_addr) { b_reg = pload1<vec>(b_addr); } template <int nelems> - EIGEN_ALWAYS_INLINE void c_store(Scalar *mem, vec &src) { + EIGEN_ALWAYS_INLINE void c_store(Scalar* mem, vec& src) { if (is_unit_inc) { switch (nelems * sizeof(*mem) * 8) { default: @@ -196,7 +196,7 @@ } template <int nelems> - EIGEN_ALWAYS_INLINE void vaddm(vec &dst, const Scalar *mem, vec &src, vec ®) { + EIGEN_ALWAYS_INLINE void vaddm(vec& dst, const Scalar* mem, vec& src, vec& reg) { if (is_unit_inc) { switch (nelems * sizeof(*mem) * 8) { default: @@ -263,7 +263,7 @@ } } - EIGEN_STRONG_INLINE void vfmadd(vec &dst, const vec &src1, const vec &src2) { + EIGEN_STRONG_INLINE void vfmadd(vec& dst, const vec& src1, const vec& src2) { dst = pmadd(src1, src2, dst); #if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0) @@ -273,7 +273,7 @@ } template <int nelems> - EIGEN_ALWAYS_INLINE void vfmaddm(vec &dst, const Scalar *mem, vec &src, vec &scale, vec ®) { + EIGEN_ALWAYS_INLINE void vfmaddm(vec& dst, const Scalar* mem, vec& src, vec& scale, vec& reg) { if (is_unit_inc) { switch (nelems * sizeof(*mem) * 8) { default: @@ -350,16 +350,16 @@ } template <int j, int endX, int i, int endY, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(j > endX) || (i > endY)> a_loads(const Scalar *ao) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(j > endX) || (i > endY)> a_loads(const Scalar* ao) { EIGEN_UNUSED_VARIABLE(ao); } template <int j, int endX, int i, int endY, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(j <= endX) && (i <= endY)> a_loads(const Scalar *ao) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(j <= endX) && (i <= endY)> a_loads(const Scalar* ao) { if (j < endX) { if (i < endY) { - auto &a_reg = zmm[a_regs[i + (j % 2) * 3]]; - const Scalar *a_addr = ao + nelems * j + nelems_in_cache_line * i - a_shift; + auto& a_reg = zmm[a_regs[i + (j % 2) * 3]]; + const Scalar* a_addr = ao + nelems * j + nelems_in_cache_line * i - a_shift; a_load<nelems>(a_reg, a_addr); a_loads<j, endX, i + 1, endY, nelems>(ao); @@ -370,8 +370,8 @@ } template <int un, int max_b_unroll, int i, int um_vecs, int a_unroll, int b_unroll> - EIGEN_ALWAYS_INLINE std::enable_if_t<(un > max_b_unroll) || (i > um_vecs)> prefetch_cs(const Scalar *co1, - const Scalar *co2) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(un > max_b_unroll) || (i > um_vecs)> prefetch_cs(const Scalar* co1, + const Scalar* co2) { EIGEN_UNUSED_VARIABLE(co1); EIGEN_UNUSED_VARIABLE(co2); } @@ -391,13 +391,13 @@ */ template <int un, int max_b_unroll, int i, int um_vecs, int a_unroll, int b_unroll> - EIGEN_ALWAYS_INLINE std::enable_if_t<(un <= max_b_unroll) && (i <= um_vecs)> prefetch_cs(Scalar *&co1, Scalar *&co2) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(un <= max_b_unroll) && (i <= um_vecs)> prefetch_cs(Scalar*& co1, Scalar*& co2) { if (un < max_b_unroll) { if (b_unroll >= un + 1) { if (un == 4 && i == 0) co2 = co1 + 4 * ldc; if (i < um_vecs) { - Scalar *co = (un + 1 <= 4) ? co1 : co2; + Scalar* co = (un + 1 <= 4) ? co1 : co2; auto co_off = (un % 4) * ldc + a_unroll - 1 + i * nelems_in_cache_line * sizeof *co; prefetch_c(co + co_off); @@ -414,16 +414,16 @@ // load_c template <int i, int um_vecs, int idx, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> scale_load_c(const Scalar *cox, vec &alpha_reg) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> scale_load_c(const Scalar* cox, vec& alpha_reg) { EIGEN_UNUSED_VARIABLE(cox); EIGEN_UNUSED_VARIABLE(alpha_reg); } template <int i, int um_vecs, int idx, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> scale_load_c(const Scalar *cox, vec &alpha_reg) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> scale_load_c(const Scalar* cox, vec& alpha_reg) { if (i < um_vecs) { - auto &c_reg = zmm[c_regs[i + idx * 3]]; - auto &c_load_reg = zmm[c_load_regs[i % 3]]; + auto& c_reg = zmm[c_regs[i + idx * 3]]; + auto& c_load_reg = zmm[c_load_regs[i % 3]]; auto c_mem = cox; if (is_unit_inc) c_mem += i * nelems_in_cache_line; @@ -443,14 +443,14 @@ // store_c template <int i, int um_vecs, int idx, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> write_c(Scalar *cox) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> write_c(Scalar* cox) { EIGEN_UNUSED_VARIABLE(cox); } template <int i, int um_vecs, int idx, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> write_c(Scalar *cox) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> write_c(Scalar* cox) { if (i < um_vecs) { - auto &c_reg = zmm[c_regs[i + idx * 3]]; + auto& c_reg = zmm[c_regs[i + idx * 3]]; auto c_mem = cox; if (is_unit_inc) c_mem += i * nelems_in_cache_line; @@ -495,20 +495,20 @@ */ template <int pow, int a_unroll, int idx> - EIGEN_ALWAYS_INLINE void c_update_1count(Scalar *&cox) { + EIGEN_ALWAYS_INLINE void c_update_1count(Scalar*& cox) { if (pow >= 4) cox += ldc; const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); - auto &alpha_reg = zmm[alpha_load_reg]; + auto& alpha_reg = zmm[alpha_load_reg]; scale_load_c<0, um_vecs, idx, a_unroll>(cox, alpha_reg); write_c<0, um_vecs, idx, a_unroll>(cox); } template <int pow, int a_unroll> - EIGEN_ALWAYS_INLINE void c_update_1pow(Scalar *&co1, Scalar *&co2) { + EIGEN_ALWAYS_INLINE void c_update_1pow(Scalar*& co1, Scalar*& co2) { constexpr int idx = pow / 2; - Scalar *&cox = idx == 0 ? co1 : co2; + Scalar*& cox = idx == 0 ? co1 : co2; constexpr int max_count = (pow + 1) / 2; static_assert(max_count <= 4, "Unsupported max_count."); @@ -520,8 +520,8 @@ } template <int max_b_unroll, int a_unroll, int b_unroll> - EIGEN_ALWAYS_INLINE void c_update(Scalar *&co1, Scalar *&co2) { - auto &alpha_reg = zmm[alpha_load_reg]; + EIGEN_ALWAYS_INLINE void c_update(Scalar*& co1, Scalar*& co2) { + auto& alpha_reg = zmm[alpha_load_reg]; co2 = co1 + ldc; if (!is_alpha1) alpha_reg = pload1<vec>(alpha); @@ -542,8 +542,8 @@ // compute template <int um, int um_vecs, int idx, int uk, bool fetch_x, bool ktail> - EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> compute(const Scalar *ao, const Scalar *bo, int &fetchA_idx, - int &fetchB_idx, vec &b_reg) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> compute(const Scalar* ao, const Scalar* bo, int& fetchA_idx, + int& fetchB_idx, vec& b_reg) { EIGEN_UNUSED_VARIABLE(ao); EIGEN_UNUSED_VARIABLE(bo); EIGEN_UNUSED_VARIABLE(fetchA_idx); @@ -552,11 +552,11 @@ } template <int um, int um_vecs, int idx, int uk, bool fetch_x, bool ktail> - EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> compute(const Scalar *ao, const Scalar *bo, int &fetchA_idx, - int &fetchB_idx, vec &b_reg) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> compute(const Scalar* ao, const Scalar* bo, int& fetchA_idx, + int& fetchB_idx, vec& b_reg) { if (um < um_vecs) { - auto &c_reg = zmm[c_regs[um + idx * 3]]; - auto &a_reg = zmm[a_regs[um + (uk % 2) * 3]]; + auto& c_reg = zmm[c_regs[um + idx * 3]]; + auto& a_reg = zmm[a_regs[um + (uk % 2) * 3]]; vfmadd(c_reg, a_reg, b_reg); @@ -578,25 +578,25 @@ // load_a template <int um, int um_vecs, int uk, int nelems, bool ktail> - EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> load_a(const Scalar *ao) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> load_a(const Scalar* ao) { EIGEN_UNUSED_VARIABLE(ao); } template <int um, int um_vecs, int uk, int nelems, bool ktail> - EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> load_a(const Scalar *ao) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> load_a(const Scalar* ao) { if (um < um_vecs) { - auto &a_reg = zmm[a_regs[um + (uk % 2) * 3]]; - const Scalar *a_addr = ao + nelems * (1 + !ktail * !use_less_a_regs + uk) + nelems_in_cache_line * um - a_shift; + auto& a_reg = zmm[a_regs[um + (uk % 2) * 3]]; + const Scalar* a_addr = ao + nelems * (1 + !ktail * !use_less_a_regs + uk) + nelems_in_cache_line * um - a_shift; a_load<nelems>(a_reg, a_addr); load_a<um + 1, um_vecs, uk, nelems, ktail>(ao); } } template <int uk, int pow, int count, int um_vecs, int b_unroll, bool ktail, bool fetch_x, bool c_fetch> - EIGEN_ALWAYS_INLINE std::enable_if_t<(count > (pow + 1) / 2)> innerkernel_1pow(const Scalar *&aa, - const Scalar *const &ao, - const Scalar *const &bo, Scalar *&co2, - int &fetchA_idx, int &fetchB_idx) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(count > (pow + 1) / 2)> innerkernel_1pow(const Scalar*& aa, + const Scalar* const& ao, + const Scalar* const& bo, Scalar*& co2, + int& fetchA_idx, int& fetchB_idx) { EIGEN_UNUSED_VARIABLE(aa); EIGEN_UNUSED_VARIABLE(ao); EIGEN_UNUSED_VARIABLE(bo); @@ -606,14 +606,14 @@ } template <int uk, int pow, int count, int um_vecs, int b_unroll, bool ktail, bool fetch_x, bool c_fetch> - EIGEN_ALWAYS_INLINE std::enable_if_t<(count <= (pow + 1) / 2)> innerkernel_1pow(const Scalar *&aa, - const Scalar *const &ao, - const Scalar *const &bo, Scalar *&co2, - int &fetchA_idx, int &fetchB_idx) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(count <= (pow + 1) / 2)> innerkernel_1pow(const Scalar*& aa, + const Scalar* const& ao, + const Scalar* const& bo, Scalar*& co2, + int& fetchA_idx, int& fetchB_idx) { const int idx = (pow / 2) + count; if (count < (pow + 1) / 2) { - auto &b_reg = zmm[b_regs[idx % 2]]; + auto& b_reg = zmm[b_regs[idx % 2]]; if (fetch_x && uk == 3 && idx == 0) prefetch_x(aa); if (fetch_x && uk == 3 && idx == 4) aa += 8; @@ -621,7 +621,7 @@ if (b_unroll >= pow) { compute<0, um_vecs, idx, uk, fetch_x, ktail>(ao, bo, fetchA_idx, fetchB_idx, b_reg); - const Scalar *b_addr = bo + b_unroll * uk + idx + 1 + (b_unroll > 1) * !use_less_b_regs - b_shift; + const Scalar* b_addr = bo + b_unroll * uk + idx + 1 + (b_unroll > 1) * !use_less_b_regs - b_shift; b_load(b_reg, b_addr); } @@ -643,8 +643,8 @@ template <int uk, int max_b_unroll, int a_unroll, int b_unroll, bool ktail, bool fetch_x, bool c_fetch, bool no_a_preload = false> - EIGEN_ALWAYS_INLINE void innerkernel_1uk(const Scalar *&aa, const Scalar *const &ao, const Scalar *const &bo, - Scalar *&co2, int &fetchA_idx, int &fetchB_idx) { + EIGEN_ALWAYS_INLINE void innerkernel_1uk(const Scalar*& aa, const Scalar* const& ao, const Scalar* const& bo, + Scalar*& co2, int& fetchA_idx, int& fetchB_idx) { const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); if (max_b_unroll >= 1) @@ -701,7 +701,7 @@ template <int a_unroll, int b_unroll, int k_factor, int max_b_unroll, int max_k_factor, bool c_fetch, bool no_a_preload = false> - EIGEN_ALWAYS_INLINE void innerkernel(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co2) { + EIGEN_ALWAYS_INLINE void innerkernel(const Scalar*& aa, const Scalar*& ao, const Scalar*& bo, Scalar*& co2) { int fetchA_idx = 0; int fetchB_idx = 0; @@ -731,7 +731,7 @@ } template <int a_unroll, int b_unroll, int max_b_unroll> - EIGEN_ALWAYS_INLINE void kloop(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { + EIGEN_ALWAYS_INLINE void kloop(const Scalar*& aa, const Scalar*& ao, const Scalar*& bo, Scalar*& co1, Scalar*& co2) { const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); if (!use_less_a_regs && k > 1) a_loads<0, 2, 0, um_vecs, a_unroll>(ao); @@ -795,7 +795,7 @@ } template <int a_unroll, int b_unroll, int max_b_unroll> - EIGEN_ALWAYS_INLINE void nloop(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { + EIGEN_ALWAYS_INLINE void nloop(const Scalar*& aa, const Scalar*& ao, const Scalar*& bo, Scalar*& co1, Scalar*& co2) { // Set A matrix pointer. ao = a + a_off * a_unroll; @@ -812,9 +812,9 @@ } template <int a_unroll, int max_a_unroll, int max_b_unroll> - EIGEN_ALWAYS_INLINE void mloop(const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { + EIGEN_ALWAYS_INLINE void mloop(const Scalar*& ao, const Scalar*& bo, Scalar*& co1, Scalar*& co2) { // Set prefetch A pointers. - const Scalar *aa = a + a_unroll * a_stride; + const Scalar* aa = a + a_unroll * a_stride; // Set C matrix pointers. co1 = c; @@ -856,10 +856,10 @@ a -= -a_shift; b -= -b_shift; - const Scalar *ao = nullptr; - const Scalar *bo = nullptr; - Scalar *co1 = nullptr; - Scalar *co2 = nullptr; + const Scalar* ao = nullptr; + const Scalar* bo = nullptr; + Scalar* co1 = nullptr; + Scalar* co2 = nullptr; // Main m-loop. for (; m >= max_a_unroll; m -= max_a_unroll) mloop<max_a_unroll, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); @@ -883,8 +883,8 @@ } } - gemm_class(Index m_, Index n_, Index k_, Index ldc_, Index inc_, const Scalar *alpha_, const Scalar *a_, - const Scalar *b_, Scalar *c_, bool is_alpha1_, bool is_beta0_, Index a_stride_, Index b_stride_, + gemm_class(Index m_, Index n_, Index k_, Index ldc_, Index inc_, const Scalar* alpha_, const Scalar* a_, + const Scalar* b_, Scalar* c_, bool is_alpha1_, bool is_beta0_, Index a_stride_, Index b_stride_, Index a_off_, Index b_off_) : m(m_), n(n_), @@ -937,8 +937,8 @@ // max_a_unroll: 24, 16, 8, 4, 2, 1 // max_b_unroll: 8, 4, 2, 1 template <typename Scalar, int max_a_unroll, int max_b_unroll, bool is_alpha1, bool is_beta0, bool is_unit_inc> -EIGEN_DONT_INLINE void gemm_kern_avx512(Index m, Index n, Index k, Scalar *alpha, const Scalar *a, const Scalar *b, - Scalar *c, Index ldc, Index inc = 1, Index a_stride = -1, Index b_stride = -1, +EIGEN_DONT_INLINE void gemm_kern_avx512(Index m, Index n, Index k, Scalar* alpha, const Scalar* a, const Scalar* b, + Scalar* c, Index ldc, Index inc = 1, Index a_stride = -1, Index b_stride = -1, Index a_off = 0, Index b_off = 0) { if (a_stride == -1) a_stride = k; if (b_stride == -1) b_stride = k; @@ -972,13 +972,13 @@ typedef typename packet_traits<Scalar>::type Packet; typedef typename DataMapper::LinearMapper LinearMapper; enum { PacketSize = packet_traits<Scalar>::size }; - EIGEN_DONT_INLINE void operator()(Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride = 0, + EIGEN_DONT_INLINE void operator()(Scalar* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0); }; template <typename Scalar, typename Index, typename DataMapper, bool Conjugate, bool PanelMode> EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Conjugate, PanelMode>::operator()( - Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride, Index offset) { + Scalar* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { constexpr int nr = 8; EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK RHS COLMAJOR"); EIGEN_UNUSED_VARIABLE(stride); @@ -1106,7 +1106,7 @@ HalfPacketSize = unpacket_traits<HalfPacket>::size, QuarterPacketSize = unpacket_traits<QuarterPacket>::size }; - EIGEN_DONT_INLINE void operator()(Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride = 0, + EIGEN_DONT_INLINE void operator()(Scalar* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0) { constexpr int nr = 8; EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK RHS ROWMAJOR"); @@ -1205,33 +1205,32 @@ template <typename Scalar, typename Index, typename DataMapper, int mr, bool ConjugateLhs, bool ConjugateRhs> struct gebp_kernel<Scalar, Scalar, Index, DataMapper, mr, 8, ConjugateLhs, ConjugateRhs> { - EIGEN_ALWAYS_INLINE void operator()(const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, + EIGEN_ALWAYS_INLINE void operator()(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA = -1, Index strideB = -1, Index offsetA = 0, Index offsetB = 0); }; template <typename Scalar, typename Index, typename DataMapper, int mr, bool ConjugateLhs, bool ConjugateRhs> EIGEN_ALWAYS_INLINE void gebp_kernel<Scalar, Scalar, Index, DataMapper, mr, 8, ConjugateLhs, ConjugateRhs>::operator()( - const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, Index cols, + const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { if (res.incr() == 1) { if (alpha == 1) { - gemm_kern_avx512<Scalar, mr, 8, true, false, true>(rows, cols, depth, &alpha, blockA, blockB, - (Scalar *)res.data(), res.stride(), res.incr(), strideA, - strideB, offsetA, offsetB); + gemm_kern_avx512<Scalar, mr, 8, true, false, true>(rows, cols, depth, &alpha, blockA, blockB, (Scalar*)res.data(), + res.stride(), res.incr(), strideA, strideB, offsetA, offsetB); } else { gemm_kern_avx512<Scalar, mr, 8, false, false, true>(rows, cols, depth, &alpha, blockA, blockB, - (Scalar *)res.data(), res.stride(), res.incr(), strideA, + (Scalar*)res.data(), res.stride(), res.incr(), strideA, strideB, offsetA, offsetB); } } else { if (alpha == 1) { gemm_kern_avx512<Scalar, mr, 8, true, false, false>(rows, cols, depth, &alpha, blockA, blockB, - (Scalar *)res.data(), res.stride(), res.incr(), strideA, + (Scalar*)res.data(), res.stride(), res.incr(), strideA, strideB, offsetA, offsetB); } else { gemm_kern_avx512<Scalar, mr, 8, false, false, false>(rows, cols, depth, &alpha, blockA, blockB, - (Scalar *)res.data(), res.stride(), res.incr(), strideA, + (Scalar*)res.data(), res.stride(), res.incr(), strideA, strideB, offsetA, offsetB); } }
diff --git a/Eigen/src/Core/arch/AVX512/PacketMath.h b/Eigen/src/Core/arch/AVX512/PacketMath.h index c8017e4..ddc766b 100644 --- a/Eigen/src/Core/arch/AVX512/PacketMath.h +++ b/Eigen/src/Core/arch/AVX512/PacketMath.h
@@ -100,7 +100,6 @@ HasCos = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, - HasBlend = 0 }; }; #endif @@ -118,7 +117,6 @@ HasMin = 1, HasMax = 1, HasConj = 1, - HasBlend = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasACos = 1, @@ -151,7 +149,6 @@ Vectorizable = 1, AlignedOnScalar = 1, size = 8, - HasBlend = 1, HasSqrt = 1, HasRsqrt = 1, HasCbrt = 1, @@ -176,7 +173,7 @@ struct packet_traits<int> : default_packet_traits { typedef Packet16i type; typedef Packet8i half; - enum { Vectorizable = 1, AlignedOnScalar = 1, HasBlend = 0, HasCmp = 1, HasDiv = 1, size = 16 }; + enum { Vectorizable = 1, AlignedOnScalar = 1, HasCmp = 1, HasDiv = 1, size = 16 }; }; template <> @@ -2512,7 +2509,6 @@ Vectorizable = 1, AlignedOnScalar = 1, size = 16, - HasBlend = 0, HasInsert = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH,
diff --git a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h index b8fa208..e6b8d99 100644 --- a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h +++ b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h
@@ -59,7 +59,6 @@ HasCos = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = 0, // EIGEN_FAST_MATH, - HasBlend = 0 }; };
diff --git a/Eigen/src/Core/arch/AVX512/TrsmKernel.h b/Eigen/src/Core/arch/AVX512/TrsmKernel.h index cb72152..11c0bd7 100644 --- a/Eigen/src/Core/arch/AVX512/TrsmKernel.h +++ b/Eigen/src/Core/arch/AVX512/TrsmKernel.h
@@ -44,7 +44,7 @@ namespace Eigen { namespace internal { -#if (EIGEN_USE_AVX512_TRSM_KERNELS) && (EIGEN_COMP_CLANG != 0) +#if (EIGEN_USE_AVX512_TRSM_KERNELS) #define EIGEN_AVX_MAX_NUM_ACC (int64_t(24)) #define EIGEN_AVX_MAX_NUM_ROW (int64_t(8)) // Denoted L in code. @@ -60,6 +60,8 @@ // Note: this depends on macros and typedefs above. #include "TrsmUnrolls.inc" +#if (EIGEN_COMP_CLANG != 0) + /** * For smaller problem sizes, and certain compilers, using the optimized kernels trsmKernelL/R directly * is faster than the packed versions in TriangularSolverMatrix.h. @@ -119,7 +121,7 @@ * Used by gemmKernel for the case A/B row-major and C col-major. */ template <typename Scalar, typename vec, int64_t unrollM, int64_t unrollN, bool remM, bool remN> -EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> &zmm, Scalar *C_arr, +EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS>& zmm, Scalar* C_arr, int64_t LDC, int64_t remM_ = 0, int64_t remN_ = 0) { EIGEN_UNUSED_VARIABLE(remN_); EIGEN_UNUSED_VARIABLE(remM_); @@ -219,7 +221,7 @@ * handleKRem: Handle arbitrary K? This is not needed for trsm. */ template <typename Scalar, bool isARowMajor, bool isCRowMajor, bool isAdd, bool handleKRem> -void gemmKernel(Scalar *A_arr, Scalar *B_arr, Scalar *C_arr, int64_t M, int64_t N, int64_t K, int64_t LDA, int64_t LDB, +void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t N, int64_t K, int64_t LDA, int64_t LDB, int64_t LDC) { using urolls = unrolls::gemm<Scalar, isAdd>; constexpr int64_t U3 = urolls::PacketSize * 3; @@ -262,8 +264,8 @@ } } if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<3, 4>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -292,8 +294,8 @@ i += 4; } if (M - i >= 2) { - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<3, 2>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -322,8 +324,8 @@ i += 2; } if (M - i > 0) { - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<3, 1>(zmm); { @@ -385,8 +387,8 @@ } } if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<2, 4>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -415,8 +417,8 @@ i += 4; } if (M - i >= 2) { - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<2, 2>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -445,8 +447,8 @@ i += 2; } if (M - i > 0) { - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<2, 1>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -506,8 +508,8 @@ } } if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 4>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -536,8 +538,8 @@ i += 4; } if (M - i >= 2) { - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 2>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -566,8 +568,8 @@ i += 2; } if (M - i > 0) { - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 1>(zmm); { @@ -601,8 +603,8 @@ constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 1; int64_t i = 0; for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, EIGEN_AVX_MAX_NUM_ROW>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -630,8 +632,8 @@ } } if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 4>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -660,8 +662,8 @@ i += 4; } if (M - i >= 2) { - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 2>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -690,8 +692,8 @@ i += 2; } if (M - i > 0) { - Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar *B_t = &B_arr[0 * LDB + j]; + Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar* B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 1>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -730,7 +732,7 @@ * The B matrix (RHS) is assumed to be row-major */ template <typename Scalar, typename vec, int64_t unrollM, bool isARowMajor, bool isFWDSolve, bool isUnitDiag> -EIGEN_ALWAYS_INLINE void triSolveKernel(Scalar *A_arr, Scalar *B_arr, int64_t K, int64_t LDA, int64_t LDB) { +EIGEN_ALWAYS_INLINE void triSolveKernel(Scalar* A_arr, Scalar* B_arr, int64_t K, int64_t LDA, int64_t LDB) { static_assert(unrollM <= EIGEN_AVX_MAX_NUM_ROW, "unrollM should be equal to EIGEN_AVX_MAX_NUM_ROW"); using urolls = unrolls::trsm<Scalar>; constexpr int64_t U3 = urolls::PacketSize * 3; @@ -780,7 +782,7 @@ * The B matrix (RHS) is assumed to be row-major */ template <typename Scalar, bool isARowMajor, bool isFWDSolve, bool isUnitDiag> -void triSolveKernelLxK(Scalar *A_arr, Scalar *B_arr, int64_t M, int64_t K, int64_t LDA, int64_t LDB) { +void triSolveKernelLxK(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t K, int64_t LDA, int64_t LDB) { // Note: this assumes EIGEN_AVX_MAX_NUM_ROW = 8. Unrolls should be adjusted // accordingly if EIGEN_AVX_MAX_NUM_ROW is smaller. using vec = typename std::conditional<std::is_same<Scalar, float>::value, vecFullFloat, vecFullDouble>::type; @@ -811,7 +813,7 @@ * */ template <typename Scalar, bool toTemp = true, bool remM = false> -EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar *B_arr, int64_t LDB, int64_t K, Scalar *B_temp, int64_t LDB_, +EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar* B_arr, int64_t LDB, int64_t K, Scalar* B_temp, int64_t LDB_, int64_t remM_ = 0) { EIGEN_UNUSED_VARIABLE(remM_); using urolls = unrolls::transB<Scalar>; @@ -898,7 +900,7 @@ */ template <typename Scalar, bool isARowMajor = true, bool isBRowMajor = true, bool isFWDSolve = true, bool isUnitDiag = false> -void triSolve(Scalar *A_arr, Scalar *B_arr, int64_t M, int64_t numRHS, int64_t LDA, int64_t LDB) { +void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t LDA, int64_t LDB) { constexpr int64_t psize = packet_traits<Scalar>::size; /** * The values for kB, numM were determined experimentally. @@ -917,7 +919,7 @@ constexpr int64_t numM = 8 * EIGEN_AVX_MAX_NUM_ROW; int64_t sizeBTemp = 0; - Scalar *B_temp = NULL; + Scalar* B_temp = NULL; EIGEN_IF_CONSTEXPR(!isBRowMajor) { /** * If B is col-major, we copy it to a fixed-size temporary array of size at most ~numM*kB and @@ -927,7 +929,7 @@ sizeBTemp = (((std::min(kB, numRHS) + psize - 1) / psize + 4) * psize) * numM; } - EIGEN_IF_CONSTEXPR(!isBRowMajor) B_temp = (Scalar *)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64); + EIGEN_IF_CONSTEXPR(!isBRowMajor) B_temp = (Scalar*)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64); for (int64_t k = 0; k < numRHS; k += kB) { int64_t bK = numRHS - k > kB ? kB : numRHS - k; @@ -1061,7 +1063,6 @@ } // Template specializations of trsmKernelL/R for float/double and inner strides of 1. -#if (EIGEN_USE_AVX512_TRSM_KERNELS) #if (EIGEN_USE_AVX512_TRSM_R_KERNELS) template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder, int OtherInnerStride, bool Specialized> @@ -1069,19 +1070,19 @@ template <typename Index, int Mode, int TriStorageOrder> struct trsmKernelR<float, Index, Mode, false, TriStorageOrder, 1, true> { - static void kernel(Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + static void kernel(Index size, Index otherSize, const float* _tri, Index triStride, float* _other, Index otherIncr, Index otherStride); }; template <typename Index, int Mode, int TriStorageOrder> struct trsmKernelR<double, Index, Mode, false, TriStorageOrder, 1, true> { - static void kernel(Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + static void kernel(Index size, Index otherSize, const double* _tri, Index triStride, double* _other, Index otherIncr, Index otherStride); }; template <typename Index, int Mode, int TriStorageOrder> EIGEN_DONT_INLINE void trsmKernelR<float, Index, Mode, false, TriStorageOrder, 1, true>::kernel( - Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + Index size, Index otherSize, const float* _tri, Index triStride, float* _other, Index otherIncr, Index otherStride) { EIGEN_UNUSED_VARIABLE(otherIncr); #ifdef EIGEN_RUNTIME_NO_MALLOC @@ -1092,12 +1093,12 @@ } #endif triSolve<float, TriStorageOrder != RowMajor, true, (Mode & Lower) != Lower, (Mode & UnitDiag) != 0>( - const_cast<float *>(_tri), _other, size, otherSize, triStride, otherStride); + const_cast<float*>(_tri), _other, size, otherSize, triStride, otherStride); } template <typename Index, int Mode, int TriStorageOrder> EIGEN_DONT_INLINE void trsmKernelR<double, Index, Mode, false, TriStorageOrder, 1, true>::kernel( - Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + Index size, Index otherSize, const double* _tri, Index triStride, double* _other, Index otherIncr, Index otherStride) { EIGEN_UNUSED_VARIABLE(otherIncr); #ifdef EIGEN_RUNTIME_NO_MALLOC @@ -1108,7 +1109,7 @@ } #endif triSolve<double, TriStorageOrder != RowMajor, true, (Mode & Lower) != Lower, (Mode & UnitDiag) != 0>( - const_cast<double *>(_tri), _other, size, otherSize, triStride, otherStride); + const_cast<double*>(_tri), _other, size, otherSize, triStride, otherStride); } #endif // (EIGEN_USE_AVX512_TRSM_R_KERNELS) @@ -1120,19 +1121,19 @@ template <typename Index, int Mode, int TriStorageOrder> struct trsmKernelL<float, Index, Mode, false, TriStorageOrder, 1, true> { - static void kernel(Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + static void kernel(Index size, Index otherSize, const float* _tri, Index triStride, float* _other, Index otherIncr, Index otherStride); }; template <typename Index, int Mode, int TriStorageOrder> struct trsmKernelL<double, Index, Mode, false, TriStorageOrder, 1, true> { - static void kernel(Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + static void kernel(Index size, Index otherSize, const double* _tri, Index triStride, double* _other, Index otherIncr, Index otherStride); }; template <typename Index, int Mode, int TriStorageOrder> EIGEN_DONT_INLINE void trsmKernelL<float, Index, Mode, false, TriStorageOrder, 1, true>::kernel( - Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + Index size, Index otherSize, const float* _tri, Index triStride, float* _other, Index otherIncr, Index otherStride) { EIGEN_UNUSED_VARIABLE(otherIncr); #ifdef EIGEN_RUNTIME_NO_MALLOC @@ -1143,12 +1144,12 @@ } #endif triSolve<float, TriStorageOrder == RowMajor, false, (Mode & Lower) == Lower, (Mode & UnitDiag) != 0>( - const_cast<float *>(_tri), _other, size, otherSize, triStride, otherStride); + const_cast<float*>(_tri), _other, size, otherSize, triStride, otherStride); } template <typename Index, int Mode, int TriStorageOrder> EIGEN_DONT_INLINE void trsmKernelL<double, Index, Mode, false, TriStorageOrder, 1, true>::kernel( - Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + Index size, Index otherSize, const double* _tri, Index triStride, double* _other, Index otherIncr, Index otherStride) { EIGEN_UNUSED_VARIABLE(otherIncr); #ifdef EIGEN_RUNTIME_NO_MALLOC @@ -1159,10 +1160,12 @@ } #endif triSolve<double, TriStorageOrder == RowMajor, false, (Mode & Lower) == Lower, (Mode & UnitDiag) != 0>( - const_cast<double *>(_tri), _other, size, otherSize, triStride, otherStride); + const_cast<double*>(_tri), _other, size, otherSize, triStride, otherStride); } #endif // EIGEN_USE_AVX512_TRSM_L_KERNELS + #endif // EIGEN_USE_AVX512_TRSM_KERNELS + } // namespace internal } // namespace Eigen #endif // EIGEN_CORE_ARCH_AVX512_TRSM_KERNEL_H
diff --git a/Eigen/src/Core/arch/GPU/PacketMath.h b/Eigen/src/Core/arch/GPU/PacketMath.h index 328b1b9..7d062f3 100644 --- a/Eigen/src/Core/arch/GPU/PacketMath.h +++ b/Eigen/src/Core/arch/GPU/PacketMath.h
@@ -84,7 +84,6 @@ HasIGammac = 1, HasBetaInc = 1, - HasBlend = 0, HasFloor = 1, HasCmp = EIGEN_HAS_GPU_DEVICE_FUNCTIONS }; @@ -117,7 +116,6 @@ HasGammaSampleDerAlpha = 1, HasIGammac = 1, HasBetaInc = 1, - HasBlend = 0, }; };
diff --git a/Eigen/src/Core/arch/HVX/PacketMath.h b/Eigen/src/Core/arch/HVX/PacketMath.h index d9f5b07..db543e0 100644 --- a/Eigen/src/Core/arch/HVX/PacketMath.h +++ b/Eigen/src/Core/arch/HVX/PacketMath.h
@@ -157,10 +157,7 @@ HasMax = 1, HasConj = 0, HasSetLinear = 0, - HasBlend = 0, - HasDiv = 0, - HasSin = 0, HasCos = 0, HasACos = 0,
diff --git a/Eigen/src/Core/arch/LSX/PacketMath.h b/Eigen/src/Core/arch/LSX/PacketMath.h index 77ffdd1..366399f 100644 --- a/Eigen/src/Core/arch/LSX/PacketMath.h +++ b/Eigen/src/Core/arch/LSX/PacketMath.h
@@ -1380,6 +1380,47 @@ } template <> +EIGEN_STRONG_INLINE Packet16c pabsdiff(const Packet16c& a, const Packet16c& b) { + return __lsx_vabsd_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pabsdiff(const Packet8s& a, const Packet8s& b) { + return __lsx_vabsd_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pabsdiff(const Packet4i& a, const Packet4i& b) { + return __lsx_vabsd_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pabsdiff(const Packet2l& a, const Packet2l& b) { + return __lsx_vabsd_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pabsdiff(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vabsd_bu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pabsdiff(const Packet8us& a, const Packet8us& b) { + return __lsx_vabsd_hu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pabsdiff(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vabsd_wu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pabsdiff(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vabsd_du(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4f pabsdiff(const Packet4f& a, const Packet4f& b) { + return pabs(psub(a, b)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pabsdiff(const Packet2d& a, const Packet2d& b) { + return pabs(psub(a, b)); +} + +template <> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return (Packet4f)__lsx_vld(from, 0); } @@ -2667,11 +2708,6 @@ return v; } template <> -EIGEN_STRONG_INLINE Packet4f pabsdiff<Packet4f>(const Packet4f& a, const Packet4f& b) { - Packet4f v = psub(a, b); - return pabs(v); -} -template <> EIGEN_STRONG_INLINE Packet4f pmin<PropagateNaN, Packet4f>(const Packet4f& a, const Packet4f& b) { return pmin<Packet4f>(a, b); } @@ -2733,48 +2769,23 @@ EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) { return pldexp_generic(a, exponent); } - -template <> -EIGEN_STRONG_INLINE Packet16c pabsdiff<Packet16c>(const Packet16c& a, const Packet16c& b) { - Packet16c v = psub(a, b); - return pabs(v); -} - -template <> -EIGEN_STRONG_INLINE Packet8s pabsdiff<Packet8s>(const Packet8s& a, const Packet8s& b) { - Packet8s v = psub(a, b); - return pabs(v); -} template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(const Packet8s& mask, const Packet8s& a, const Packet8s& b) { return __lsx_vbitsel_v(b, a, mask); } - -template <> -EIGEN_STRONG_INLINE Packet4i pabsdiff<Packet4i>(const Packet4i& a, const Packet4i& b) { - Packet4i v = psub(a, b); - return pabs(v); -} template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b) { return __lsx_vbitsel_v(b, a, mask); } - template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b) { return __lsx_vbitsel_v(b, a, mask); } - template <> EIGEN_STRONG_INLINE Packet16uc pdiv<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return __lsx_vdiv_bu(a, b); } template <> -EIGEN_STRONG_INLINE Packet16uc pabsdiff<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { - Packet16uc v = psub(a, b); - return pabs(v); -} -template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(const Packet16uc& mask, const Packet16uc& a, const Packet16uc& b) { return __lsx_vbitsel_v(b, a, mask); @@ -2791,12 +2802,6 @@ } return res; } - -template <> -EIGEN_STRONG_INLINE Packet8us pabsdiff<Packet8us>(const Packet8us& a, const Packet8us& b) { - Packet8us v = psub(a, b); - return pabs(v); -} template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(const Packet8us& mask, const Packet8us& a, const Packet8us& b) { return __lsx_vbitsel_v(b, a, mask); @@ -2815,11 +2820,6 @@ } template <> -EIGEN_STRONG_INLINE Packet4ui pabsdiff<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { - Packet4ui v = psub(a, b); - return pabs(v); -} -template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b) { return __lsx_vbitsel_v(b, a, mask); }
diff --git a/Eigen/src/Core/arch/SSE/PacketMath.h b/Eigen/src/Core/arch/SSE/PacketMath.h index b8b1d9a..1ea23b0 100644 --- a/Eigen/src/Core/arch/SSE/PacketMath.h +++ b/Eigen/src/Core/arch/SSE/PacketMath.h
@@ -200,7 +200,6 @@ HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, HasErfc = EIGEN_FAST_MATH, - HasBlend = 1, HasSign = 0 // The manually vectorized version is slightly slower for SSE. }; }; @@ -230,7 +229,6 @@ HasCbrt = 1, HasATan = 1, HasATanh = 1, - HasBlend = 1 }; }; template <> @@ -245,7 +243,6 @@ HasCmp = 1, HasDiv = 1, HasShift = 1, - HasBlend = 1 }; }; template <> @@ -257,11 +254,9 @@ AlignedOnScalar = 1, size = 4, - HasDiv = 0, HasNegate = 0, HasCmp = 1, HasShift = 1, - HasBlend = 1 }; }; template <> @@ -273,10 +268,8 @@ AlignedOnScalar = 1, size = 2, - HasDiv = 0, HasCmp = 1, HasShift = 1, - HasBlend = 1 }; }; #endif
diff --git a/Eigen/src/Core/arch/SYCL/InteropHeaders.h b/Eigen/src/Core/arch/SYCL/InteropHeaders.h index 578e0f3..0bdf825 100644 --- a/Eigen/src/Core/arch/SYCL/InteropHeaders.h +++ b/Eigen/src/Core/arch/SYCL/InteropHeaders.h
@@ -30,7 +30,7 @@ namespace internal { -template <int has_blend, int lengths> +template <int lengths> struct sycl_packet_traits : default_packet_traits { enum { Vectorizable = 1, @@ -60,7 +60,6 @@ HasIGamma = 0, HasIGammac = 0, HasBetaInc = 0, - HasBlend = has_blend, // This flag is used to indicate whether packet comparison is supported. // pcmp_eq, pcmp_lt and pcmp_le should be defined for it to be true. HasCmp = 1, @@ -78,19 +77,19 @@ }; #ifdef SYCL_DEVICE_ONLY -#define SYCL_PACKET_TRAITS(packet_type, has_blend, unpacket_type, lengths) \ - template <> \ - struct packet_traits<unpacket_type> : sycl_packet_traits<has_blend, lengths> { \ - typedef packet_type type; \ - typedef packet_type half; \ +#define SYCL_PACKET_TRAITS(packet_type, unpacket_type, lengths) \ + template <> \ + struct packet_traits<unpacket_type> : sycl_packet_traits<lengths> { \ + typedef packet_type type; \ + typedef packet_type half; \ }; -SYCL_PACKET_TRAITS(cl::sycl::cl_half8, 1, Eigen::half, 8) -SYCL_PACKET_TRAITS(cl::sycl::cl_half8, 1, const Eigen::half, 8) -SYCL_PACKET_TRAITS(cl::sycl::cl_float4, 1, float, 4) -SYCL_PACKET_TRAITS(cl::sycl::cl_float4, 1, const float, 4) -SYCL_PACKET_TRAITS(cl::sycl::cl_double2, 0, double, 2) -SYCL_PACKET_TRAITS(cl::sycl::cl_double2, 0, const double, 2) +SYCL_PACKET_TRAITS(cl::sycl::cl_half8, Eigen::half, 8) +SYCL_PACKET_TRAITS(cl::sycl::cl_half8, const Eigen::half, 8) +SYCL_PACKET_TRAITS(cl::sycl::cl_float4, float, 4) +SYCL_PACKET_TRAITS(cl::sycl::cl_float4, const float, 4) +SYCL_PACKET_TRAITS(cl::sycl::cl_double2, double, 2) +SYCL_PACKET_TRAITS(cl::sycl::cl_double2, const double, 2) #undef SYCL_PACKET_TRAITS // Make sure this is only available when targeting a GPU: we don't want to @@ -135,14 +134,14 @@ struct PacketWrapper { typedef typename ::Eigen::internal::unpacket_traits<PacketReturnType>::type Scalar; template <typename Index> - EIGEN_DEVICE_FUNC static Scalar scalarize(Index, PacketReturnType &) { + EIGEN_DEVICE_FUNC static Scalar scalarize(Index, PacketReturnType&) { eigen_assert(false && "THERE IS NO PACKETIZE VERSION FOR THE CHOSEN TYPE"); abort(); } EIGEN_DEVICE_FUNC static PacketReturnType convert_to_packet_type(Scalar in, Scalar) { return ::Eigen::internal::template plset<PacketReturnType>(in); } - EIGEN_DEVICE_FUNC static void set_packet(PacketReturnType, Scalar *) { + EIGEN_DEVICE_FUNC static void set_packet(PacketReturnType, Scalar*) { eigen_assert(false && "THERE IS NO PACKETIZE VERSION FOR THE CHOSEN TYPE"); abort(); } @@ -153,7 +152,7 @@ struct PacketWrapper<PacketReturnType, 4> { typedef typename ::Eigen::internal::unpacket_traits<PacketReturnType>::type Scalar; template <typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index index, PacketReturnType &in) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index index, PacketReturnType& in) { switch (index) { case 0: return in.x(); @@ -174,7 +173,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType convert_to_packet_type(Scalar in, Scalar other) { return PacketReturnType(in, other, other, other); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType &lhs, Scalar *rhs) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType& lhs, Scalar* rhs) { lhs = PacketReturnType(rhs[0], rhs[1], rhs[2], rhs[3]); } }; @@ -183,20 +182,20 @@ struct PacketWrapper<PacketReturnType, 1> { typedef typename ::Eigen::internal::unpacket_traits<PacketReturnType>::type Scalar; template <typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index, PacketReturnType &in) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index, PacketReturnType& in) { return in; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType convert_to_packet_type(Scalar in, Scalar) { return PacketReturnType(in); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType &lhs, Scalar *rhs) { lhs = rhs[0]; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType& lhs, Scalar* rhs) { lhs = rhs[0]; } }; template <typename PacketReturnType> struct PacketWrapper<PacketReturnType, 2> { typedef typename ::Eigen::internal::unpacket_traits<PacketReturnType>::type Scalar; template <typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index index, PacketReturnType &in) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index index, PacketReturnType& in) { switch (index) { case 0: return in.x(); @@ -213,7 +212,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType convert_to_packet_type(Scalar in, Scalar other) { return PacketReturnType(in, other); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType &lhs, Scalar *rhs) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType& lhs, Scalar* rhs) { lhs = PacketReturnType(rhs[0], rhs[1]); } };
diff --git a/Eigen/src/Core/arch/clang/PacketMath.h b/Eigen/src/Core/arch/clang/PacketMath.h index 359f1e4..55ddc1c 100644 --- a/Eigen/src/Core/arch/clang/PacketMath.h +++ b/Eigen/src/Core/arch/clang/PacketMath.h
@@ -31,13 +31,9 @@ using Packet8l = detail::VectorType<int64_t, 8>; // --- packet_traits specializations --- -template <> -struct packet_traits<float> : default_packet_traits { - using type = Packet16f; - using half = Packet16f; +struct generic_float_packet_traits : default_packet_traits { enum { Vectorizable = 1, - size = 16, AlignedOnScalar = 1, HasAdd = 1, HasSub = 1, @@ -46,7 +42,8 @@ HasNegate = 1, HasAbs = 1, HasRound = 1, - HasMinMax = 1, + HasMin = 1, + HasMax = 1, HasCmp = 1, HasSet1 = 1, HasCast = 1, @@ -80,12 +77,24 @@ }; template <> -struct packet_traits<double> : default_packet_traits { +struct packet_traits<float> : generic_float_packet_traits { + using type = Packet16f; + using half = Packet16f; + enum { + size = 16, + }; +}; + +template <> +struct packet_traits<double> : generic_float_packet_traits { using type = Packet8d; using half = Packet8d; + enum { size = 8, HasACos = 0, HasASin = 0 }; +}; + +struct generic_integer_packet_traits : default_packet_traits { enum { Vectorizable = 1, - size = 8, AlignedOnScalar = 1, HasAdd = 1, HasSub = 1, @@ -93,157 +102,85 @@ HasDiv = 1, HasNegate = 1, HasAbs = 1, - HasRound = 1, - HasMinMax = 1, + HasMin = 1, + HasMax = 1, HasCmp = 1, HasSet1 = 1, HasCast = 1, HasBitwise = 1, HasRedux = 1, - HasSign = 1, + // Set remaining to 0 + HasRound = 1, + HasSqrt = 0, + HasRsqrt = 0, + HasReciprocal = 0, HasArg = 0, HasConj = 1, - // Math functions - HasReciprocal = 1, - HasSin = 1, - HasCos = 1, - HasACos = 0, - HasASin = 0, - HasATan = 1, - HasATanh = 1, - HasLog = 1, - HasLog1p = 1, - HasExpm1 = 1, - HasExp = 1, - HasPow = 1, - HasNdtri = 1, - HasBessel = 1, - HasSqrt = 1, - HasRsqrt = 1, - HasCbrt = 1, - HasTanh = 1, - HasErf = 1, - HasErfc = 1 + HasExp = 0, + HasLog = 0, + HasSin = 0, + HasCos = 0, }; }; template <> -struct packet_traits<int32_t> : default_packet_traits { +struct packet_traits<int32_t> : generic_integer_packet_traits { using type = Packet16i; using half = Packet16i; enum { - Vectorizable = 1, size = 16, - AlignedOnScalar = 1, - HasAdd = 1, - HasSub = 1, - HasMul = 1, - HasDiv = 1, - HasNegate = 1, - HasAbs = 1, - HasMinMax = 1, - HasCmp = 1, - HasSet1 = 1, - HasCast = 1, - HasBitwise = 1, - HasRedux = 1, - // Set remaining to 0 - HasRound = 1, - HasSqrt = 0, - HasRsqrt = 0, - HasReciprocal = 0, - HasArg = 0, - HasConj = 1, - HasExp = 0, - HasLog = 0, - HasSin = 0, - HasCos = 0, }; }; template <> -struct packet_traits<int64_t> : default_packet_traits { +struct packet_traits<int64_t> : generic_integer_packet_traits { using type = Packet8l; using half = Packet8l; enum { - Vectorizable = 1, size = 8, - AlignedOnScalar = 1, - HasAdd = 1, - HasSub = 1, - HasMul = 1, - HasDiv = 1, - HasNegate = 1, - HasAbs = 1, - HasMinMax = 1, - HasCmp = 1, - HasSet1 = 1, - HasCast = 1, - HasBitwise = 1, - HasRedux = 1, - // Set remaining to 0 - HasRound = 1, - HasSqrt = 0, - HasRsqrt = 0, - HasReciprocal = 0, - HasArg = 0, - HasConj = 1, - HasExp = 0, - HasLog = 0, - HasSin = 0, - HasCos = 0, }; }; // --- unpacket_traits specializations --- +struct generic_unpacket_traits : default_unpacket_traits { + enum { + alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, + vectorizable = true, + }; +}; + template <> -struct unpacket_traits<Packet16f> { +struct unpacket_traits<Packet16f> : generic_unpacket_traits { using type = float; using half = Packet16f; using integer_packet = Packet16i; enum { size = 16, - alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, - vectorizable = true, - masked_load_available = false, - masked_store_available = false }; }; template <> -struct unpacket_traits<Packet8d> { +struct unpacket_traits<Packet8d> : generic_unpacket_traits { using type = double; using half = Packet8d; using integer_packet = Packet8l; enum { size = 8, - alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, - vectorizable = true, - masked_load_available = false, - masked_store_available = false }; }; template <> -struct unpacket_traits<Packet16i> { +struct unpacket_traits<Packet16i> : generic_unpacket_traits { using type = int32_t; using half = Packet16i; enum { size = 16, - alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, - vectorizable = true, - masked_load_available = false, - masked_store_available = false }; }; template <> -struct unpacket_traits<Packet8l> { +struct unpacket_traits<Packet8l> : generic_unpacket_traits { using type = int64_t; using half = Packet8l; enum { size = 8, - alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, - vectorizable = true, - masked_load_available = false, - masked_store_available = false }; }; @@ -458,8 +395,8 @@ #undef EIGEN_CLANG_PACKET_BITWISE_FLOAT // --- Min/Max operations --- -#if __has_builtin(__builtin_elementwise_min) && __has_builtin(__builtin_elementwise_max) && \ - __has_builtin(__builtin_elementwise_abs) +#if EIGEN_HAS_BUILTIN(__builtin_elementwise_min) && EIGEN_HAS_BUILTIN(__builtin_elementwise_max) && \ + EIGEN_HAS_BUILTIN(__builtin_elementwise_abs) #define EIGEN_CLANG_PACKET_ELEMENTWISE(PACKET_TYPE) \ template <> \ EIGEN_STRONG_INLINE PACKET_TYPE pmin<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ @@ -494,7 +431,7 @@ template <> \ EIGEN_STRONG_INLINE PACKET_TYPE pselect<PACKET_TYPE>(const PACKET_TYPE& mask, const PACKET_TYPE& a, \ const PACKET_TYPE& b) { \ - return __builtin_elementwise_abs(mask) == 0 ? b : a; \ + return mask != 0 ? a : b; \ } EIGEN_CLANG_PACKET_ELEMENTWISE(Packet16f) @@ -506,9 +443,9 @@ // --- Math functions (float/double only) --- -#if __has_builtin(__builtin_elementwise_floor) && __has_builtin(__builtin_elementwise_ceil) && \ - __has_builtin(__builtin_elementwise_round) && __has_builtin(__builtin_elementwise_roundeven) && \ - __has_builtin(__builtin_elementwise_trunc) && __has_builtin(__builtin_elementwise_sqrt) +#if EIGEN_HAS_BUILTIN(__builtin_elementwise_floor) && EIGEN_HAS_BUILTIN(__builtin_elementwise_ceil) && \ + EIGEN_HAS_BUILTIN(__builtin_elementwise_round) && EIGEN_HAS_BUILTIN(__builtin_elementwise_roundeven) && \ + EIGEN_HAS_BUILTIN(__builtin_elementwise_trunc) && EIGEN_HAS_BUILTIN(__builtin_elementwise_sqrt) #define EIGEN_CLANG_PACKET_MATH_FLOAT(PACKET_TYPE) \ template <> \ EIGEN_STRONG_INLINE PACKET_TYPE pfloor<PACKET_TYPE>(const PACKET_TYPE& a) { \ @@ -541,7 +478,7 @@ #endif // --- Fused Multiply-Add (MADD) --- -#if defined(__FMA__) && __has_builtin(__builtin_elementwise_fma) +#if defined(__FMA__) && EIGEN_HAS_BUILTIN(__builtin_elementwise_fma) #define EIGEN_CLANG_PACKET_MADD(PACKET_TYPE) \ template <> \ EIGEN_STRONG_INLINE PACKET_TYPE pmadd<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \ @@ -589,11 +526,11 @@ EIGEN_STRONG_INLINE PACKET_TYPE pgather<typename unpacket_traits<PACKET_TYPE>::type, PACKET_TYPE>( \ const unpacket_traits<PACKET_TYPE>::type* from, Index stride) { \ constexpr int size = unpacket_traits<PACKET_TYPE>::size; \ - unpacket_traits<PACKET_TYPE>::type arr[size]; \ + PACKET_TYPE result; \ for (int i = 0; i < size; ++i) { \ - arr[i] = from[i * stride]; \ + result[i] = from[i * stride]; \ } \ - return *reinterpret_cast<PACKET_TYPE*>(arr); \ + return result; \ } EIGEN_CLANG_PACKET_SCATTER_GATHER(Packet16f) @@ -603,7 +540,7 @@ #undef EIGEN_CLANG_PACKET_SCATTER_GATHER // ---- Various operations that depend on __builtin_shufflevector. -#if __has_builtin(__builtin_shufflevector) +#if EIGEN_HAS_BUILTIN(__builtin_shufflevector) namespace detail { template <typename Packet> EIGEN_STRONG_INLINE Packet preverse_impl_8(const Packet& a) {
diff --git a/Eigen/src/Core/arch/clang/Reductions.h b/Eigen/src/Core/arch/clang/Reductions.h index 8454a9a..d9cc558 100644 --- a/Eigen/src/Core/arch/clang/Reductions.h +++ b/Eigen/src/Core/arch/clang/Reductions.h
@@ -14,7 +14,8 @@ namespace internal { // --- Reductions --- -#if __has_builtin(__builtin_reduce_min) && __has_builtin(__builtin_reduce_max) && __has_builtin(__builtin_reduce_or) +#if EIGEN_HAS_BUILTIN(__builtin_reduce_min) && EIGEN_HAS_BUILTIN(__builtin_reduce_max) && \ + EIGEN_HAS_BUILTIN(__builtin_reduce_or) #define EIGEN_CLANG_PACKET_REDUX_MINMAX(PACKET_TYPE) \ template <> \ EIGEN_STRONG_INLINE unpacket_traits<PACKET_TYPE>::type predux_min(const PACKET_TYPE& a) { \ @@ -36,7 +37,7 @@ #undef EIGEN_CLANG_PACKET_REDUX_MINMAX #endif -#if __has_builtin(__builtin_reduce_add) && __has_builtin(__builtin_reduce_mul) +#if EIGEN_HAS_BUILTIN(__builtin_reduce_add) && EIGEN_HAS_BUILTIN(__builtin_reduce_mul) #define EIGEN_CLANG_PACKET_REDUX_INT(PACKET_TYPE) \ template <> \ EIGEN_STRONG_INLINE unpacket_traits<PACKET_TYPE>::type predux<PACKET_TYPE>(const PACKET_TYPE& a) { \ @@ -53,7 +54,7 @@ #undef EIGEN_CLANG_PACKET_REDUX_INT #endif -#if __has_builtin(__builtin_shufflevector) +#if EIGEN_HAS_BUILTIN(__builtin_shufflevector) namespace detail { template <typename VectorT> EIGEN_STRONG_INLINE scalar_type_of_vector_t<VectorT> ReduceAdd16(const VectorT& a) {
diff --git a/Eigen/src/Core/arch/clang/TypeCasting.h b/Eigen/src/Core/arch/clang/TypeCasting.h index 0aa4496..164056b 100644 --- a/Eigen/src/Core/arch/clang/TypeCasting.h +++ b/Eigen/src/Core/arch/clang/TypeCasting.h
@@ -37,7 +37,7 @@ //============================================================================== // pcast //============================================================================== -#if __has_builtin(__builtin_convertvector) +#if EIGEN_HAS_BUILTIN(__builtin_convertvector) template <> EIGEN_STRONG_INLINE Packet16i pcast<Packet16f, Packet16i>(const Packet16f& a) { return __builtin_convertvector(a, Packet16i);
diff --git a/Eigen/src/Core/functors/UnaryFunctors.h b/Eigen/src/Core/functors/UnaryFunctors.h index 202995f..d7fc7bb 100644 --- a/Eigen/src/Core/functors/UnaryFunctors.h +++ b/Eigen/src/Core/functors/UnaryFunctors.h
@@ -106,26 +106,6 @@ }; }; -template <typename Scalar, bool IsComplex = NumTraits<Scalar>::IsComplex> -struct squared_norm_functor { - typedef Scalar result_type; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator()(const Scalar& a) const { - return Scalar(numext::real(a) * numext::real(a), numext::imag(a) * numext::imag(a)); - } - template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { - return Packet(pmul(a.v, a.v)); - } -}; -template <typename Scalar> -struct squared_norm_functor<Scalar, false> : scalar_abs2_op<Scalar> {}; - -template <typename Scalar> -struct functor_traits<squared_norm_functor<Scalar>> { - using Real = typename NumTraits<Scalar>::Real; - enum { Cost = NumTraits<Real>::MulCost, PacketAccess = packet_traits<Real>::HasMul }; -}; - /** \internal * \brief Template functor to compute the conjugate of a complex value *
diff --git a/Eigen/src/Core/util/ForwardDeclarations.h b/Eigen/src/Core/util/ForwardDeclarations.h index e0bc57e..e2e7155 100644 --- a/Eigen/src/Core/util/ForwardDeclarations.h +++ b/Eigen/src/Core/util/ForwardDeclarations.h
@@ -517,6 +517,9 @@ template <typename Packet> struct has_packet_segment : std::false_type {}; + +template <typename T> +struct complex_array_access; } // namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/util/Macros.h b/Eigen/src/Core/util/Macros.h index cb00082..2cec29b 100644 --- a/Eigen/src/Core/util/Macros.h +++ b/Eigen/src/Core/util/Macros.h
@@ -704,6 +704,13 @@ #define EIGEN_HAS_BUILTIN(x) 0 #endif +// Cross compiler wrapper around LLVM's __has_attribute +#ifdef __has_attribute +#define EIGEN_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +#define EIGEN_HAS_ATTRIBUTE(x) 0 +#endif + // A Clang feature extension to determine compiler features. // We use it to determine 'cxx_rvalue_references' #ifndef __has_feature @@ -835,7 +842,7 @@ #endif // Does the compiler support vector types? -#if __has_attribute(ext_vector_type) && __has_builtin(__builtin_vectorelements) +#if EIGEN_HAS_ATTRIBUTE(ext_vector_type) && EIGEN_HAS_BUILTIN(__builtin_vectorelements) #define EIGEN_ARCH_VECTOR_EXTENSIONS 1 #else #define EIGEN_ARCH_VECTOR_EXTENSIONS 0
diff --git a/Eigen/src/SVD/JacobiSVD.h b/Eigen/src/SVD/JacobiSVD.h index dfcb6df..7c52a35 100644 --- a/Eigen/src/SVD/JacobiSVD.h +++ b/Eigen/src/SVD/JacobiSVD.h
@@ -720,6 +720,7 @@ m_isInitialized = true; m_info = InvalidInput; m_nonzeroSingularValues = 0; + m_singularValues.setZero(); return *this; } if (numext::is_exactly_zero(scale)) scale = RealScalar(1);
diff --git a/test/rand.cpp b/test/rand.cpp index 4131f38..69e7cf9 100644 --- a/test/rand.cpp +++ b/test/rand.cpp
@@ -30,12 +30,10 @@ template <typename Scalar> void check_all_in_range(Scalar x, Scalar y) { constexpr int repeats = 32; - uint64_t count = static_cast<uint64_t>(y) - static_cast<uint64_t>(x) + 1; + Index count = static_cast<Index>(y) - static_cast<Index>(x) + 1; ArrayX<bool> mask(count); - // ensure that `count` does not overflow the return type of `mask.size()` - VERIFY(count == static_cast<uint64_t>(mask.size())); mask.setConstant(false); - for (uint64_t k = 0; k < count; k++) + for (Index k = 0; k < count; k++) for (int repeat = 0; repeat < repeats; repeat++) { Scalar r = check_in_range(x, y); Index i = static_cast<Index>(r) - static_cast<Index>(x);
diff --git a/test/realview.cpp b/test/realview.cpp index 8658a3f..7cefe6c 100644 --- a/test/realview.cpp +++ b/test/realview.cpp
@@ -9,8 +9,20 @@ #include "main.h" +// wrapper that disables array-oriented access to real and imaginary components +struct TestComplex : public std::complex<float> { + TestComplex() = default; + TestComplex(const TestComplex&) = default; + TestComplex(std::complex<float> x) : std::complex<float>(x){}; + TestComplex(float x) : std::complex<float>(x){}; +}; +template <> +struct NumTraits<TestComplex> : NumTraits<std::complex<float>> {}; +template <> +struct internal::random_impl<TestComplex> : internal::random_impl<std::complex<float>> {}; + template <typename T> -void test_realview(const T&) { +void test_realview_readonly(const T&) { using Scalar = typename T::Scalar; using RealScalar = typename NumTraits<Scalar>::Real; @@ -26,21 +38,16 @@ Index rows = internal::random<Index>(minRows, maxRows); Index cols = internal::random<Index>(minCols, maxCols); - T A(rows, cols), B, C; + T A(rows, cols), B(rows, cols); VERIFY(A.realView().rows() == rowFactor * A.rows()); VERIFY(A.realView().cols() == colFactor * A.cols()); VERIFY(A.realView().size() == sizeFactor * A.size()); - RealScalar alpha = internal::random(RealScalar(1), RealScalar(2)); A.setRandom(); + VERIFY_IS_APPROX(A.matrix().cwiseAbs2().sum(), A.realView().matrix().cwiseAbs2().sum()); - VERIFY_IS_APPROX(A.matrix().squaredNorm(), A.realView().matrix().squaredNorm()); - - // test re-sizing realView during assignment - B.realView() = A.realView(); - VERIFY_IS_APPROX(A, B); - VERIFY_IS_APPROX(A.realView(), B.realView()); + RealScalar alpha = internal::random(RealScalar(1), RealScalar(2)); // B = A * alpha for (Index r = 0; r < rows; r++) { @@ -48,14 +55,7 @@ B.coeffRef(r, c) = A.coeff(r, c) * Scalar(alpha); } } - - VERIFY_IS_APPROX(B.realView(), A.realView() * alpha); - C = A; - C.realView() *= alpha; - VERIFY_IS_APPROX(B, C); - - alpha = internal::random(RealScalar(1), RealScalar(2)); - A.setRandom(); + VERIFY_IS_CWISE_APPROX(B.realView(), A.realView() * alpha); // B = A / alpha for (Index r = 0; r < rows; r++) { @@ -63,15 +63,155 @@ B.coeffRef(r, c) = A.coeff(r, c) / Scalar(alpha); } } + VERIFY_IS_CWISE_APPROX(B.realView(), A.realView() / alpha); +} +template <typename T> +void test_realview(const T&) { + using Scalar = typename T::Scalar; + using RealScalar = typename NumTraits<Scalar>::Real; + + constexpr Index minRows = T::RowsAtCompileTime == Dynamic ? 1 : T::RowsAtCompileTime; + constexpr Index maxRows = T::MaxRowsAtCompileTime == Dynamic ? (EIGEN_TEST_MAX_SIZE / 2) : T::MaxRowsAtCompileTime; + constexpr Index minCols = T::ColsAtCompileTime == Dynamic ? 1 : T::ColsAtCompileTime; + constexpr Index maxCols = T::MaxColsAtCompileTime == Dynamic ? (EIGEN_TEST_MAX_SIZE / 2) : T::MaxColsAtCompileTime; + + constexpr Index rowFactor = (NumTraits<Scalar>::IsComplex && !T::IsRowMajor) ? 2 : 1; + constexpr Index colFactor = (NumTraits<Scalar>::IsComplex && T::IsRowMajor) ? 2 : 1; + constexpr Index sizeFactor = NumTraits<Scalar>::IsComplex ? 2 : 1; + + const Index rows = internal::random<Index>(minRows, maxRows); + const Index cols = internal::random<Index>(minCols, maxCols); + const Index realViewRows = rowFactor * rows; + const Index realViewCols = colFactor * cols; + + const T A = T::Random(rows, cols); + T B; + + VERIFY_IS_EQUAL(A.realView().rows(), rowFactor * A.rows()); + VERIFY_IS_EQUAL(A.realView().cols(), colFactor * A.cols()); + VERIFY_IS_EQUAL(A.realView().size(), sizeFactor * A.size()); + + VERIFY_IS_APPROX(A.matrix().cwiseAbs2().sum(), A.realView().matrix().cwiseAbs2().sum()); + + // test re-sizing realView during assignment + B.realView() = A.realView(); + VERIFY_IS_APPROX(A, B); + VERIFY_IS_APPROX(A.realView(), B.realView()); + + const RealScalar alpha = internal::random(RealScalar(1), RealScalar(2)); + + // B = A * alpha + for (Index r = 0; r < rows; r++) { + for (Index c = 0; c < cols; c++) { + B.coeffRef(r, c) = A.coeff(r, c) * Scalar(alpha); + } + } + VERIFY_IS_APPROX(B.realView(), A.realView() * alpha); + + B = A; + B.realView() *= alpha; + VERIFY_IS_APPROX(B.realView(), A.realView() * alpha); + + // B = A / alpha + for (Index r = 0; r < rows; r++) { + for (Index c = 0; c < cols; c++) { + B.coeffRef(r, c) = A.coeff(r, c) / Scalar(alpha); + } + } VERIFY_IS_APPROX(B.realView(), A.realView() / alpha); + + B = A; + B.realView() /= alpha; + VERIFY_IS_APPROX(B.realView(), A.realView() / alpha); + + // force some usual access patterns + Index malloc_size = (rows * cols * sizeof(Scalar)) + sizeof(RealScalar); + void* data1 = internal::aligned_malloc(malloc_size); + void* data2 = internal::aligned_malloc(malloc_size); + Scalar* ptr1 = reinterpret_cast<Scalar*>(reinterpret_cast<uint8_t*>(data1) + sizeof(RealScalar)); + Scalar* ptr2 = reinterpret_cast<Scalar*>(reinterpret_cast<uint8_t*>(data2) + sizeof(RealScalar)); + Map<T> C(ptr1, rows, cols), D(ptr2, rows, cols); + + C.setRandom(); + D.setRandom(); + for (Index r = 0; r < realViewRows; r++) { + for (Index c = 0; c < realViewCols; c++) { + C.realView().coeffRef(r, c) = D.realView().coeff(r, c); + } + } + VERIFY_IS_CWISE_EQUAL(C, D); + C = A; - C.realView() /= alpha; - VERIFY_IS_APPROX(B, C); + + for (Index c = 0; c < realViewCols - 1; c++) { + B.realView().row(0).coeffRef(realViewCols - 1 - c) = C.realView().row(0).coeff(c + 1); + } + D.realView().row(0).tail(realViewCols - 1) = C.realView().row(0).tail(realViewCols - 1).reverse(); + VERIFY_IS_CWISE_EQUAL(B.realView().row(0).tail(realViewCols - 1), D.realView().row(0).tail(realViewCols - 1)); + + for (Index r = 0; r < realViewRows - 1; r++) { + B.realView().col(0).coeffRef(realViewRows - 1 - r) = C.realView().col(0).coeff(r + 1); + } + D.realView().col(0).tail(realViewRows - 1) = C.realView().col(0).tail(realViewRows - 1).reverse(); + VERIFY_IS_CWISE_EQUAL(B.realView().col(0).tail(realViewRows - 1), D.realView().col(0).tail(realViewRows - 1)); +} + +template <typename ComplexScalar, bool Enable = internal::packet_traits<ComplexScalar>::Vectorizable> +struct test_edge_cases_impl { + static void run() { + using namespace internal; + using RealScalar = typename NumTraits<ComplexScalar>::Real; + using ComplexPacket = typename packet_traits<ComplexScalar>::type; + using RealPacket = typename unpacket_traits<ComplexPacket>::as_real; + constexpr int ComplexSize = unpacket_traits<ComplexPacket>::size; + constexpr int RealSize = 2 * ComplexSize; + VectorX<ComplexScalar> a_data(2 * ComplexSize); + Map<const VectorX<RealScalar>> a_data_asreal(reinterpret_cast<const RealScalar*>(a_data.data()), 2 * a_data.size()); + VectorX<RealScalar> b_data(RealSize); + + a_data.setRandom(); + evaluator<RealView<VectorX<ComplexScalar>>> eval(a_data.realView()); + + for (Index offset = 0; offset < RealSize; offset++) { + for (Index begin = 0; offset + begin < RealSize; begin++) { + for (Index count = 0; begin + count < RealSize; count++) { + b_data.setRandom(); + RealPacket res = eval.template packetSegment<Unaligned, RealPacket>(offset, begin, count); + pstoreSegment(b_data.data(), res, begin, count); + VERIFY_IS_CWISE_EQUAL(a_data_asreal.segment(offset + begin, count), b_data.segment(begin, count)); + } + } + } + } +}; + +template <typename ComplexScalar> +struct test_edge_cases_impl<ComplexScalar, false> { + static void run() {} +}; + +template <typename ComplexScalar> +void test_edge_cases(const ComplexScalar&) { + test_edge_cases_impl<ComplexScalar>::run(); } template <typename Scalar, int Rows, int Cols, int MaxRows = Rows, int MaxCols = Cols> -void test_realview_driver() { +void test_realview_readonly() { + // if Rows == 1, don't test ColMajor as it is not a valid array + using ColMajorMatrixType = Matrix<Scalar, Rows, Cols, Rows == 1 ? RowMajor : ColMajor, MaxRows, MaxCols>; + using ColMajorArrayType = Array<Scalar, Rows, Cols, Rows == 1 ? RowMajor : ColMajor, MaxRows, MaxCols>; + // if Cols == 1, don't test RowMajor as it is not a valid array + using RowMajorMatrixType = Matrix<Scalar, Rows, Cols, Cols == 1 ? ColMajor : RowMajor, MaxRows, MaxCols>; + using RowMajorArrayType = Array<Scalar, Rows, Cols, Cols == 1 ? ColMajor : RowMajor, MaxRows, MaxCols>; + test_realview_readonly(ColMajorMatrixType()); + test_realview_readonly(ColMajorArrayType()); + test_realview_readonly(RowMajorMatrixType()); + test_realview_readonly(RowMajorArrayType()); +} + +template <typename Scalar, int Rows, int Cols, int MaxRows = Rows, int MaxCols = Cols> +void test_realview_readwrite() { // if Rows == 1, don't test ColMajor as it is not a valid array using ColMajorMatrixType = Matrix<Scalar, Rows, Cols, Rows == 1 ? RowMajor : ColMajor, MaxRows, MaxCols>; using ColMajorArrayType = Array<Scalar, Rows, Cols, Rows == 1 ? RowMajor : ColMajor, MaxRows, MaxCols>; @@ -85,26 +225,29 @@ } template <int Rows, int Cols, int MaxRows = Rows, int MaxCols = Cols> -void test_realview_driver_complex() { - test_realview_driver<float, Rows, Cols, MaxRows, MaxCols>(); - test_realview_driver<std::complex<float>, Rows, Cols, MaxRows, MaxCols>(); - test_realview_driver<double, Rows, Cols, MaxRows, MaxCols>(); - test_realview_driver<std::complex<double>, Rows, Cols, MaxRows, MaxCols>(); - test_realview_driver<long double, Rows, Cols, MaxRows, MaxCols>(); - test_realview_driver<std::complex<long double>, Rows, Cols, MaxRows, MaxCols>(); +void test_realview() { + test_realview_readwrite<float, Rows, Cols, MaxRows, MaxCols>(); + test_realview_readwrite<std::complex<float>, Rows, Cols, MaxRows, MaxCols>(); + test_realview_readwrite<double, Rows, Cols, MaxRows, MaxCols>(); + test_realview_readwrite<std::complex<double>, Rows, Cols, MaxRows, MaxCols>(); + test_realview_readwrite<long double, Rows, Cols, MaxRows, MaxCols>(); + test_realview_readwrite<std::complex<long double>, Rows, Cols, MaxRows, MaxCols>(); + test_realview_readonly<TestComplex, Rows, Cols, MaxRows, MaxCols>(); } EIGEN_DECLARE_TEST(realview) { for (int i = 0; i < g_repeat; i++) { - CALL_SUBTEST_1((test_realview_driver_complex<Dynamic, Dynamic, Dynamic, Dynamic>())); - CALL_SUBTEST_2((test_realview_driver_complex<Dynamic, Dynamic, 17, Dynamic>())); - CALL_SUBTEST_3((test_realview_driver_complex<Dynamic, Dynamic, Dynamic, 19>())); - CALL_SUBTEST_4((test_realview_driver_complex<Dynamic, Dynamic, 17, 19>())); - CALL_SUBTEST_5((test_realview_driver_complex<17, Dynamic, 17, Dynamic>())); - CALL_SUBTEST_6((test_realview_driver_complex<Dynamic, 19, Dynamic, 19>())); - CALL_SUBTEST_7((test_realview_driver_complex<17, 19, 17, 19>())); - CALL_SUBTEST_8((test_realview_driver_complex<Dynamic, 1>())); - CALL_SUBTEST_9((test_realview_driver_complex<1, Dynamic>())); - CALL_SUBTEST_10((test_realview_driver_complex<1, 1>())); + CALL_SUBTEST_1((test_realview<Dynamic, Dynamic, Dynamic, Dynamic>())); + CALL_SUBTEST_2((test_realview<Dynamic, Dynamic, 17, Dynamic>())); + CALL_SUBTEST_3((test_realview<Dynamic, Dynamic, Dynamic, 19>())); + CALL_SUBTEST_4((test_realview<Dynamic, Dynamic, 17, 19>())); + CALL_SUBTEST_5((test_realview<17, Dynamic, 17, Dynamic>())); + CALL_SUBTEST_6((test_realview<Dynamic, 19, Dynamic, 19>())); + CALL_SUBTEST_7((test_realview<17, 19, 17, 19>())); + CALL_SUBTEST_8((test_realview<Dynamic, 1>())); + CALL_SUBTEST_9((test_realview<1, Dynamic>())); + CALL_SUBTEST_10((test_realview<1, 1>())); + CALL_SUBTEST_11(test_edge_cases(std::complex<float>())); + CALL_SUBTEST_12(test_edge_cases(std::complex<double>())); } }
diff --git a/test/threads_fork_join.cpp b/test/threads_fork_join.cpp index b852b05..ec75bb7 100644 --- a/test/threads_fork_join.cpp +++ b/test/threads_fork_join.cpp
@@ -26,7 +26,7 @@ TestData test_data = make_test_data(/*num_threads=*/4, kNumTasks); std::atomic<uint64_t> sum(0); std::function<void(Index, Index)> binary_do_fn = [&](Index i, Index j) { - for (int k = i; k < j; ++k) + for (Index k = i; k < j; ++k) for (uint64_t new_sum = sum; !sum.compare_exchange_weak(new_sum, new_sum + test_data.data[k]);) { }; };
diff --git a/unsupported/test/cxx11_tensor_expr.cpp b/unsupported/test/cxx11_tensor_expr.cpp index fc235d5..b9c14c3 100644 --- a/unsupported/test/cxx11_tensor_expr.cpp +++ b/unsupported/test/cxx11_tensor_expr.cpp
@@ -312,6 +312,9 @@ } } +// Nan propagation does currently not work like one would expect from std::max/std::min, +// so we disable it for now +#if !EIGEN_ARCH_ARM_OR_ARM64 template <typename Scalar> void test_minmax_nan_propagation_templ() { for (int size = 1; size < 17; ++size) { @@ -430,6 +433,10 @@ VERIFY_IS_EQUAL(val(), (size == 1 ? -kInf : kZero)); } } +#else +template <typename Scalar> +void test_minmax_nan_propagation_templ() {}; +#endif static void test_clip() { Tensor<float, 1> vec(6); @@ -465,10 +472,5 @@ CALL_SUBTEST(test_type_casting()); CALL_SUBTEST(test_select()); CALL_SUBTEST(test_clip()); - -// Nan propagation does currently not work like one would expect from std::max/std::min, -// so we disable it for now -#if !EIGEN_ARCH_ARM_OR_ARM64 CALL_SUBTEST(test_minmax_nan_propagation()); -#endif }