Update Eigen to commit:035cf684983197a13e579905d98198c365f44538 CHANGELOG ========= 035cf6849 - Fix build of realview.cpp 23a5482fc - Misc. packet math cleanups. 4cb0776f8 - Add 5.0.1 release notes and a few unreleased features. 8b85f5933 - Fix realview ffcd7bdbd - Avoid breaking the build on older compilers. da867c31c - Fix defines in AVX512 custom TRSM kernel. PiperOrigin-RevId: 829998360 Change-Id: I974aefe61d7a0071eb5e27d93f103b716e35ddf4
diff --git a/Eigen/src/Core/DenseBase.h b/Eigen/src/Core/DenseBase.h index dcadcbf..c81e1d1 100644 --- a/Eigen/src/Core/DenseBase.h +++ b/Eigen/src/Core/DenseBase.h
@@ -367,11 +367,11 @@ EIGEN_DEVICE_FUNC inline bool allFinite() const; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator*=(const Scalar& other); - template <bool Enable = internal::complex_array_access<Scalar>::value, typename = std::enable_if_t<Enable>> + template <bool Enable = !internal::is_same<Scalar, RealScalar>::value, typename = std::enable_if_t<Enable>> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator*=(const RealScalar& other); EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator/=(const Scalar& other); - template <bool Enable = internal::complex_array_access<Scalar>::value, typename = std::enable_if_t<Enable>> + template <bool Enable = !internal::is_same<Scalar, RealScalar>::value, typename = std::enable_if_t<Enable>> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator/=(const RealScalar& other); typedef internal::add_const_on_value_type_t<typename internal::eval<Derived>::type> EvalReturnType;
diff --git a/Eigen/src/Core/Dot.h b/Eigen/src/Core/Dot.h index a173306..059527c 100644 --- a/Eigen/src/Core/Dot.h +++ b/Eigen/src/Core/Dot.h
@@ -20,7 +20,10 @@ template <typename Derived, typename Scalar = typename traits<Derived>::Scalar> struct squared_norm_impl { using Real = typename NumTraits<Scalar>::Real; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Real run(const Derived& a) { return a.realView().cwiseAbs2().sum(); } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Real run(const Derived& a) { + Scalar result = a.unaryExpr(squared_norm_functor<Scalar>()).sum(); + return numext::real(result) + numext::imag(result); + } }; template <typename Derived>
diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index ba38cd1..bfb52ab 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h
@@ -57,12 +57,12 @@ HasConj = 1, HasSetLinear = 1, HasSign = 1, - HasAbsDiff = 1, // By default, the nearest integer functions (rint, round, floor, ceil, trunc) are enabled for all scalar and packet // types HasRound = 1, HasArg = 0, + HasAbsDiff = 0, // This flag is used to indicate whether packet comparison is supported. // pcmp_eq and pcmp_lt should be defined for it to be true. HasCmp = 0, @@ -116,7 +116,6 @@ enum { HasAdd = 0, HasSub = 0, - HasAbsDiff = 0, HasMul = 0, HasNegate = 0, HasAbs = 0, @@ -131,18 +130,17 @@ template <typename T> struct packet_traits<const T> : packet_traits<T> {}; -struct default_unpacket_traits { - enum { vectorizable = false, masked_load_available = false, masked_store_available = false }; -}; - template <typename T> -struct unpacket_traits : default_unpacket_traits { +struct unpacket_traits { typedef T type; typedef T half; typedef typename numext::get_integer_by_size<sizeof(T)>::signed_type integer_packet; enum { size = 1, alignment = alignof(T), + vectorizable = false, + masked_load_available = false, + masked_store_available = false }; }; @@ -749,15 +747,9 @@ /** \internal \returns the min of \a a and \a b (coeff-wise) */ template <typename Packet> -EIGEN_DEVICE_FUNC inline std::enable_if_t<NumTraits<typename unpacket_traits<Packet>::type>::IsInteger, Packet> -pabsdiff(const Packet& a, const Packet& b) { +EIGEN_DEVICE_FUNC inline Packet pabsdiff(const Packet& a, const Packet& b) { return pselect(pcmp_lt(a, b), psub(b, a), psub(a, b)); } -template <typename Packet> -EIGEN_DEVICE_FUNC inline std::enable_if_t<!NumTraits<typename unpacket_traits<Packet>::type>::IsInteger, Packet> -pabsdiff(const Packet& a, const Packet& b) { - return pabs(psub(a, b)); -} /** \internal \returns a packet version of \a *from, from must be properly aligned */ template <typename Packet>
diff --git a/Eigen/src/Core/RealView.h b/Eigen/src/Core/RealView.h index 3be5556..7ba42f9 100644 --- a/Eigen/src/Core/RealView.h +++ b/Eigen/src/Core/RealView.h
@@ -17,16 +17,20 @@ namespace internal { -// Write access and vectorization requires array-oriented access to the real and imaginary components. +// Vectorized assignment to RealView requires array-oriented access to the real and imaginary components. // From https://en.cppreference.com/w/cpp/numeric/complex.html: // For any pointer to an element of an array of std::complex<T> named p and any valid array index i, // reinterpret_cast<T*>(p)[2 * i] is the real part of the complex number p[i], and // reinterpret_cast<T*>(p)[2 * i + 1] is the imaginary part of the complex number p[i]. -template <typename T> +template <typename ComplexScalar> struct complex_array_access : std::false_type {}; -template <typename T> -struct complex_array_access<std::complex<T>> : std::true_type {}; +template <> +struct complex_array_access<std::complex<float>> : std::true_type {}; +template <> +struct complex_array_access<std::complex<double>> : std::true_type {}; +template <> +struct complex_array_access<std::complex<long double>> : std::true_type {}; template <typename Xpr> struct traits<RealView<Xpr>> : public traits<Xpr> { @@ -36,17 +40,13 @@ if (size_as_int == Dynamic) return Dynamic; return times_two ? (2 * size_as_int) : size_as_int; } - using Base = traits<Xpr>; using ComplexScalar = typename Base::Scalar; using Scalar = typename NumTraits<ComplexScalar>::Real; - - static constexpr bool ArrayAccess = complex_array_access<ComplexScalar>::value; - static constexpr int ActualDirectAccessBit = ArrayAccess ? DirectAccessBit : 0; - static constexpr int ActualLvaluebit = !std::is_const<Xpr>::value && ArrayAccess ? LvalueBit : 0; + static constexpr int ActualDirectAccessBit = complex_array_access<ComplexScalar>::value ? DirectAccessBit : 0; static constexpr int ActualPacketAccessBit = packet_traits<Scalar>::Vectorizable ? PacketAccessBit : 0; static constexpr int FlagMask = - ActualDirectAccessBit | ActualLvaluebit | ActualPacketAccessBit | HereditaryBits | LinearAccessBit; + ActualDirectAccessBit | ActualPacketAccessBit | HereditaryBits | LinearAccessBit | LvalueBit; static constexpr int BaseFlags = int(evaluator<Xpr>::Flags) | int(Base::Flags); static constexpr int Flags = BaseFlags & FlagMask; static constexpr bool IsRowMajor = Flags & RowMajorBit; @@ -66,63 +66,55 @@ using XprType = RealView<Xpr>; using ExpressionTraits = traits<XprType>; using ComplexScalar = typename ExpressionTraits::ComplexScalar; + using ComplexCoeffReturnType = typename BaseEvaluator::CoeffReturnType; using Scalar = typename ExpressionTraits::Scalar; + static constexpr bool IsRowMajor = ExpressionTraits::IsRowMajor; static constexpr int Flags = ExpressionTraits::Flags; static constexpr int CoeffReadCost = BaseEvaluator::CoeffReadCost; static constexpr int Alignment = BaseEvaluator::Alignment; - static constexpr bool IsRowMajor = ExpressionTraits::IsRowMajor; - static constexpr bool DirectAccess = Flags & DirectAccessBit; - - using ComplexCoeffReturnType = std::conditional_t<DirectAccess, const ComplexScalar&, ComplexScalar>; - using CoeffReturnType = std::conditional_t<DirectAccess, const Scalar&, Scalar>; EIGEN_DEVICE_FUNC explicit evaluator(XprType realView) : BaseEvaluator(realView.m_xpr) {} - template <bool Enable = DirectAccess, std::enable_if_t<!Enable, bool> = true> + template <bool Enable = std::is_reference<ComplexCoeffReturnType>::value, typename = std::enable_if_t<!Enable>> constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index row, Index col) const { - Index r = IsRowMajor ? row : row / 2; - Index c = IsRowMajor ? col / 2 : col; - bool p = (IsRowMajor ? col : row) & 1; - ComplexScalar ccoeff = BaseEvaluator::coeff(r, c); - return p ? numext::imag(ccoeff) : numext::real(ccoeff); - } - template <bool Enable = DirectAccess, std::enable_if_t<Enable, bool> = true> - constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { - Index r = IsRowMajor ? row : row / 2; - Index c = IsRowMajor ? col / 2 : col; + ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(IsRowMajor ? row : row / 2, IsRowMajor ? col / 2 : col); Index p = (IsRowMajor ? col : row) & 1; - ComplexCoeffReturnType ccoeff = BaseEvaluator::coeff(r, c); - return reinterpret_cast<const Scalar(&)[2]>(ccoeff)[p]; - } - template <bool Enable = DirectAccess, std::enable_if_t<!Enable, bool> = true> - constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index index) const { - ComplexScalar ccoeff = BaseEvaluator::coeff(index / 2); - bool p = index & 1; - return p ? numext::imag(ccoeff) : numext::real(ccoeff); - } - template <bool Enable = DirectAccess, std::enable_if_t<Enable, bool> = true> - constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { - ComplexCoeffReturnType ccoeff = BaseEvaluator::coeff(index / 2); - Index p = index & 1; - return reinterpret_cast<const Scalar(&)[2]>(ccoeff)[p]; - } - constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { - Index r = IsRowMajor ? row : row / 2; - Index c = IsRowMajor ? col / 2 : col; - Index p = (IsRowMajor ? col : row) & 1; - ComplexScalar& ccoeffRef = BaseEvaluator::coeffRef(r, c); - return reinterpret_cast<Scalar(&)[2]>(ccoeffRef)[p]; - } - constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { - ComplexScalar& ccoeffRef = BaseEvaluator::coeffRef(index / 2); - Index p = index & 1; - return reinterpret_cast<Scalar(&)[2]>(ccoeffRef)[p]; + return p ? numext::real(cscalar) : numext::imag(cscalar); } - // If the first index is odd (imaginary), discard the first scalar - // in 'result' and assign the missing scalar. - // This operation is safe as the real component of the first scalar must exist. + template <bool Enable = std::is_reference<ComplexCoeffReturnType>::value, typename = std::enable_if_t<Enable>> + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index row, Index col) const { + ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(IsRowMajor ? row : row / 2, IsRowMajor ? col / 2 : col); + Index p = (IsRowMajor ? col : row) & 1; + return reinterpret_cast<const Scalar(&)[2]>(cscalar)[p]; + } + + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { + ComplexScalar& cscalar = BaseEvaluator::coeffRef(IsRowMajor ? row : row / 2, IsRowMajor ? col / 2 : col); + Index p = (IsRowMajor ? col : row) & 1; + return reinterpret_cast<Scalar(&)[2]>(cscalar)[p]; + } + + template <bool Enable = std::is_reference<ComplexCoeffReturnType>::value, typename = std::enable_if_t<!Enable>> + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index index) const { + ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(index / 2); + Index p = index & 1; + return p ? numext::real(cscalar) : numext::imag(cscalar); + } + + template <bool Enable = std::is_reference<ComplexCoeffReturnType>::value, typename = std::enable_if_t<Enable>> + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index index) const { + ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(index / 2); + Index p = index & 1; + return reinterpret_cast<const Scalar(&)[2]>(cscalar)[p]; + } + + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { + ComplexScalar& cscalar = BaseEvaluator::coeffRef(index / 2); + Index p = index & 1; + return reinterpret_cast<Scalar(&)[2]>(cscalar)[p]; + } template <int LoadMode, typename PacketType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { @@ -130,20 +122,12 @@ using ComplexPacket = typename find_packet_by_size<ComplexScalar, RealPacketSize / 2>::type; EIGEN_STATIC_ASSERT((find_packet_by_size<ComplexScalar, RealPacketSize / 2>::value), MISSING COMPATIBLE COMPLEX PACKET TYPE) - Index r = IsRowMajor ? row : row / 2; - Index c = IsRowMajor ? col / 2 : col; - bool p = (IsRowMajor ? col : row) & 1; - ComplexPacket cresult = BaseEvaluator::template packet<LoadMode, ComplexPacket>(r, c); - PacketType result = preinterpret<PacketType>(cresult); - if (p) { - Scalar aux[RealPacketSize + 1]; - pstoreu(aux, result); - Index lastr = IsRowMajor ? row : row + RealPacketSize - 1; - Index lastc = IsRowMajor ? col + RealPacketSize - 1 : col; - aux[RealPacketSize] = coeff(lastr, lastc); - result = ploadu<PacketType>(aux + 1); - } - return result; + eigen_assert(((IsRowMajor ? col : row) % 2 == 0) && "the inner index must be even"); + + Index crow = IsRowMajor ? row : row / 2; + Index ccol = IsRowMajor ? col / 2 : col; + ComplexPacket cpacket = BaseEvaluator::template packet<LoadMode, ComplexPacket>(crow, ccol); + return preinterpret<PacketType, ComplexPacket>(cpacket); } template <int LoadMode, typename PacketType> @@ -152,23 +136,12 @@ using ComplexPacket = typename find_packet_by_size<ComplexScalar, RealPacketSize / 2>::type; EIGEN_STATIC_ASSERT((find_packet_by_size<ComplexScalar, RealPacketSize / 2>::value), MISSING COMPATIBLE COMPLEX PACKET TYPE) - ComplexPacket cresult = BaseEvaluator::template packet<LoadMode, ComplexPacket>(index / 2); - PacketType result = preinterpret<PacketType>(cresult); - bool p = index & 1; - if (p) { - Scalar aux[RealPacketSize + 1]; - pstoreu(aux, result); - aux[RealPacketSize] = coeff(index + RealPacketSize - 1); - result = ploadu<PacketType>(aux + 1); - } - return result; - } + eigen_assert((index % 2 == 0) && "the index must be even"); - // The requested real packet segment forms the half-open interval [begin, end), where 'end' = 'begin' + 'count'. - // In order to access the underlying complex array, even indices must be aligned with the real components - // of the complex scalars. 'begin' and 'count' must be modified as follows: - // a) 'begin' must be rounded down to the nearest even number; and - // b) 'end' must be rounded up to the nearest even number. + Index cindex = index / 2; + ComplexPacket cpacket = BaseEvaluator::template packet<LoadMode, ComplexPacket>(cindex); + return preinterpret<PacketType, ComplexPacket>(cpacket); + } template <int LoadMode, typename PacketType> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index row, Index col, Index begin, Index count) const { @@ -176,24 +149,15 @@ using ComplexPacket = typename find_packet_by_size<ComplexScalar, RealPacketSize / 2>::type; EIGEN_STATIC_ASSERT((find_packet_by_size<ComplexScalar, RealPacketSize / 2>::value), MISSING COMPATIBLE COMPLEX PACKET TYPE) - Index actualBegin = numext::round_down(begin, 2); - Index actualEnd = numext::round_down(begin + count + 1, 2); - Index actualCount = actualEnd - actualBegin; - Index r = IsRowMajor ? row : row / 2; - Index c = IsRowMajor ? col / 2 : col; - ComplexPacket cresult = - BaseEvaluator::template packetSegment<LoadMode, ComplexPacket>(r, c, actualBegin / 2, actualCount / 2); - PacketType result = preinterpret<PacketType>(cresult); - bool p = (IsRowMajor ? col : row) & 1; - if (p) { - Scalar aux[RealPacketSize + 1] = {}; - pstoreu(aux, result); - Index lastr = IsRowMajor ? row : row + actualEnd - 1; - Index lastc = IsRowMajor ? col + actualEnd - 1 : col; - aux[actualEnd] = coeff(lastr, lastc); - result = ploadu<PacketType>(aux + 1); - } - return result; + eigen_assert(((IsRowMajor ? col : row) % 2 == 0) && "the inner index must be even"); + eigen_assert((begin % 2 == 0) && (count % 2 == 0) && "begin and count must be even"); + + Index crow = IsRowMajor ? row : row / 2; + Index ccol = IsRowMajor ? col / 2 : col; + Index cbegin = begin / 2; + Index ccount = count / 2; + ComplexPacket cpacket = BaseEvaluator::template packetSegment<LoadMode, ComplexPacket>(crow, ccol, cbegin, ccount); + return preinterpret<PacketType, ComplexPacket>(cpacket); } template <int LoadMode, typename PacketType> @@ -202,20 +166,14 @@ using ComplexPacket = typename find_packet_by_size<ComplexScalar, RealPacketSize / 2>::type; EIGEN_STATIC_ASSERT((find_packet_by_size<ComplexScalar, RealPacketSize / 2>::value), MISSING COMPATIBLE COMPLEX PACKET TYPE) - Index actualBegin = numext::round_down(begin, 2); - Index actualEnd = numext::round_down(begin + count + 1, 2); - Index actualCount = actualEnd - actualBegin; - ComplexPacket cresult = - BaseEvaluator::template packetSegment<LoadMode, ComplexPacket>(index / 2, actualBegin / 2, actualCount / 2); - PacketType result = preinterpret<PacketType>(cresult); - bool p = index & 1; - if (p) { - Scalar aux[RealPacketSize + 1] = {}; - pstoreu(aux, result); - aux[actualEnd] = coeff(index + actualEnd - 1); - result = ploadu<PacketType>(aux + 1); - } - return result; + eigen_assert((index % 2 == 0) && "the index must be even"); + eigen_assert((begin % 2 == 0) && (count % 2 == 0) && "begin and count must be even"); + + Index cindex = index / 2; + Index cbegin = begin / 2; + Index ccount = count / 2; + ComplexPacket cpacket = BaseEvaluator::template packetSegment<LoadMode, ComplexPacket>(cindex, cbegin, ccount); + return preinterpret<PacketType, ComplexPacket>(cpacket); } }; @@ -253,7 +211,7 @@ EIGEN_DEVICE_FUNC RealView& operator=(const DenseBase<OtherDerived>& other); protected: - friend struct internal::evaluator<RealView>; + friend struct internal::evaluator<RealView<Xpr>>; Xpr& m_xpr; };
diff --git a/Eigen/src/Core/arch/AVX/PacketMath.h b/Eigen/src/Core/arch/AVX/PacketMath.h index 318b375..48fe815 100644 --- a/Eigen/src/Core/arch/AVX/PacketMath.h +++ b/Eigen/src/Core/arch/AVX/PacketMath.h
@@ -250,6 +250,7 @@ HasDiv = 0, HasNegate = 0, + HasSqrt = 0, HasCmp = 1, HasMin = 1, @@ -276,9 +277,12 @@ AlignedOnScalar = 1, size = 4, + // HasMin = 0, + // HasMax = 0, HasDiv = 0, HasTranspose = 0, HasNegate = 0, + HasSqrt = 0, HasCmp = 1, HasShift = 1 };
diff --git a/Eigen/src/Core/arch/AVX512/GemmKernel.h b/Eigen/src/Core/arch/AVX512/GemmKernel.h index 96157f4..483e149 100644 --- a/Eigen/src/Core/arch/AVX512/GemmKernel.h +++ b/Eigen/src/Core/arch/AVX512/GemmKernel.h
@@ -81,10 +81,10 @@ Index m; const Index n, k, ldc; const Index inc; - const Scalar* alpha; + const Scalar *alpha; const Scalar *a, *b; - Scalar* c; + Scalar *c; const bool is_alpha1; const bool is_beta0; @@ -92,26 +92,26 @@ const Index a_stride, b_stride; const Index a_off, b_off; - EIGEN_ALWAYS_INLINE void prefetch_a(const Scalar* a_addr) { - _mm_prefetch((char*)(a_prefetch_size + a_addr - a_shift), _MM_HINT_T0); + EIGEN_ALWAYS_INLINE void prefetch_a(const Scalar *a_addr) { + _mm_prefetch((char *)(a_prefetch_size + a_addr - a_shift), _MM_HINT_T0); } - EIGEN_ALWAYS_INLINE void prefetch_b(const Scalar* b_addr) { - _mm_prefetch((char*)(b_prefetch_size + b_addr - b_shift), _MM_HINT_T0); + EIGEN_ALWAYS_INLINE void prefetch_b(const Scalar *b_addr) { + _mm_prefetch((char *)(b_prefetch_size + b_addr - b_shift), _MM_HINT_T0); } - EIGEN_ALWAYS_INLINE void prefetch_x(const Scalar* x_addr) { _mm_prefetch((char*)(x_addr - a_shift), _MM_HINT_T2); } + EIGEN_ALWAYS_INLINE void prefetch_x(const Scalar *x_addr) { _mm_prefetch((char *)(x_addr - a_shift), _MM_HINT_T2); } - EIGEN_ALWAYS_INLINE void prefetch_c(const Scalar* c_addr) { + EIGEN_ALWAYS_INLINE void prefetch_c(const Scalar *c_addr) { #if defined(__PRFCHW__) && __PRFCHW__ == 1 - _m_prefetchw((void*)c_addr); + _m_prefetchw((void *)c_addr); #else - _mm_prefetch((char*)c_addr, _MM_HINT_T0); + _mm_prefetch((char *)c_addr, _MM_HINT_T0); #endif } template <int nelems> - EIGEN_ALWAYS_INLINE void a_load(vec& a_reg, const Scalar* a_addr) { + EIGEN_ALWAYS_INLINE void a_load(vec &a_reg, const Scalar *a_addr) { switch (nelems * sizeof(*a_addr) * 8) { default: case 512 * 3: @@ -124,13 +124,13 @@ a_reg = ploadu<vec>(a_addr); break; case 256 * 1: - a_reg = preinterpret<vec>(_mm512_broadcast_f64x4(ploadu<Packet4d>(reinterpret_cast<const double*>(a_addr)))); + a_reg = preinterpret<vec>(_mm512_broadcast_f64x4(ploadu<Packet4d>(reinterpret_cast<const double *>(a_addr)))); break; case 128 * 1: - a_reg = preinterpret<vec>(_mm512_broadcast_f32x4(ploadu<Packet4f>(reinterpret_cast<const float*>(a_addr)))); + a_reg = preinterpret<vec>(_mm512_broadcast_f32x4(ploadu<Packet4f>(reinterpret_cast<const float *>(a_addr)))); break; case 64 * 1: - a_reg = preinterpret<vec>(pload1<Packet8d>(reinterpret_cast<const double*>(a_addr))); + a_reg = preinterpret<vec>(pload1<Packet8d>(reinterpret_cast<const double *>(a_addr))); break; case 32 * 1: a_reg = pload1<vec>(a_addr); @@ -138,10 +138,10 @@ } } - EIGEN_ALWAYS_INLINE void b_load(vec& b_reg, const Scalar* b_addr) { b_reg = pload1<vec>(b_addr); } + EIGEN_ALWAYS_INLINE void b_load(vec &b_reg, const Scalar *b_addr) { b_reg = pload1<vec>(b_addr); } template <int nelems> - EIGEN_ALWAYS_INLINE void c_store(Scalar* mem, vec& src) { + EIGEN_ALWAYS_INLINE void c_store(Scalar *mem, vec &src) { if (is_unit_inc) { switch (nelems * sizeof(*mem) * 8) { default: @@ -196,7 +196,7 @@ } template <int nelems> - EIGEN_ALWAYS_INLINE void vaddm(vec& dst, const Scalar* mem, vec& src, vec& reg) { + EIGEN_ALWAYS_INLINE void vaddm(vec &dst, const Scalar *mem, vec &src, vec ®) { if (is_unit_inc) { switch (nelems * sizeof(*mem) * 8) { default: @@ -263,7 +263,7 @@ } } - EIGEN_STRONG_INLINE void vfmadd(vec& dst, const vec& src1, const vec& src2) { + EIGEN_STRONG_INLINE void vfmadd(vec &dst, const vec &src1, const vec &src2) { dst = pmadd(src1, src2, dst); #if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0) @@ -273,7 +273,7 @@ } template <int nelems> - EIGEN_ALWAYS_INLINE void vfmaddm(vec& dst, const Scalar* mem, vec& src, vec& scale, vec& reg) { + EIGEN_ALWAYS_INLINE void vfmaddm(vec &dst, const Scalar *mem, vec &src, vec &scale, vec ®) { if (is_unit_inc) { switch (nelems * sizeof(*mem) * 8) { default: @@ -350,16 +350,16 @@ } template <int j, int endX, int i, int endY, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(j > endX) || (i > endY)> a_loads(const Scalar* ao) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(j > endX) || (i > endY)> a_loads(const Scalar *ao) { EIGEN_UNUSED_VARIABLE(ao); } template <int j, int endX, int i, int endY, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(j <= endX) && (i <= endY)> a_loads(const Scalar* ao) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(j <= endX) && (i <= endY)> a_loads(const Scalar *ao) { if (j < endX) { if (i < endY) { - auto& a_reg = zmm[a_regs[i + (j % 2) * 3]]; - const Scalar* a_addr = ao + nelems * j + nelems_in_cache_line * i - a_shift; + auto &a_reg = zmm[a_regs[i + (j % 2) * 3]]; + const Scalar *a_addr = ao + nelems * j + nelems_in_cache_line * i - a_shift; a_load<nelems>(a_reg, a_addr); a_loads<j, endX, i + 1, endY, nelems>(ao); @@ -370,8 +370,8 @@ } template <int un, int max_b_unroll, int i, int um_vecs, int a_unroll, int b_unroll> - EIGEN_ALWAYS_INLINE std::enable_if_t<(un > max_b_unroll) || (i > um_vecs)> prefetch_cs(const Scalar* co1, - const Scalar* co2) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(un > max_b_unroll) || (i > um_vecs)> prefetch_cs(const Scalar *co1, + const Scalar *co2) { EIGEN_UNUSED_VARIABLE(co1); EIGEN_UNUSED_VARIABLE(co2); } @@ -391,13 +391,13 @@ */ template <int un, int max_b_unroll, int i, int um_vecs, int a_unroll, int b_unroll> - EIGEN_ALWAYS_INLINE std::enable_if_t<(un <= max_b_unroll) && (i <= um_vecs)> prefetch_cs(Scalar*& co1, Scalar*& co2) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(un <= max_b_unroll) && (i <= um_vecs)> prefetch_cs(Scalar *&co1, Scalar *&co2) { if (un < max_b_unroll) { if (b_unroll >= un + 1) { if (un == 4 && i == 0) co2 = co1 + 4 * ldc; if (i < um_vecs) { - Scalar* co = (un + 1 <= 4) ? co1 : co2; + Scalar *co = (un + 1 <= 4) ? co1 : co2; auto co_off = (un % 4) * ldc + a_unroll - 1 + i * nelems_in_cache_line * sizeof *co; prefetch_c(co + co_off); @@ -414,16 +414,16 @@ // load_c template <int i, int um_vecs, int idx, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> scale_load_c(const Scalar* cox, vec& alpha_reg) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> scale_load_c(const Scalar *cox, vec &alpha_reg) { EIGEN_UNUSED_VARIABLE(cox); EIGEN_UNUSED_VARIABLE(alpha_reg); } template <int i, int um_vecs, int idx, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> scale_load_c(const Scalar* cox, vec& alpha_reg) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> scale_load_c(const Scalar *cox, vec &alpha_reg) { if (i < um_vecs) { - auto& c_reg = zmm[c_regs[i + idx * 3]]; - auto& c_load_reg = zmm[c_load_regs[i % 3]]; + auto &c_reg = zmm[c_regs[i + idx * 3]]; + auto &c_load_reg = zmm[c_load_regs[i % 3]]; auto c_mem = cox; if (is_unit_inc) c_mem += i * nelems_in_cache_line; @@ -443,14 +443,14 @@ // store_c template <int i, int um_vecs, int idx, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> write_c(Scalar* cox) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> write_c(Scalar *cox) { EIGEN_UNUSED_VARIABLE(cox); } template <int i, int um_vecs, int idx, int nelems> - EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> write_c(Scalar* cox) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> write_c(Scalar *cox) { if (i < um_vecs) { - auto& c_reg = zmm[c_regs[i + idx * 3]]; + auto &c_reg = zmm[c_regs[i + idx * 3]]; auto c_mem = cox; if (is_unit_inc) c_mem += i * nelems_in_cache_line; @@ -495,20 +495,20 @@ */ template <int pow, int a_unroll, int idx> - EIGEN_ALWAYS_INLINE void c_update_1count(Scalar*& cox) { + EIGEN_ALWAYS_INLINE void c_update_1count(Scalar *&cox) { if (pow >= 4) cox += ldc; const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); - auto& alpha_reg = zmm[alpha_load_reg]; + auto &alpha_reg = zmm[alpha_load_reg]; scale_load_c<0, um_vecs, idx, a_unroll>(cox, alpha_reg); write_c<0, um_vecs, idx, a_unroll>(cox); } template <int pow, int a_unroll> - EIGEN_ALWAYS_INLINE void c_update_1pow(Scalar*& co1, Scalar*& co2) { + EIGEN_ALWAYS_INLINE void c_update_1pow(Scalar *&co1, Scalar *&co2) { constexpr int idx = pow / 2; - Scalar*& cox = idx == 0 ? co1 : co2; + Scalar *&cox = idx == 0 ? co1 : co2; constexpr int max_count = (pow + 1) / 2; static_assert(max_count <= 4, "Unsupported max_count."); @@ -520,8 +520,8 @@ } template <int max_b_unroll, int a_unroll, int b_unroll> - EIGEN_ALWAYS_INLINE void c_update(Scalar*& co1, Scalar*& co2) { - auto& alpha_reg = zmm[alpha_load_reg]; + EIGEN_ALWAYS_INLINE void c_update(Scalar *&co1, Scalar *&co2) { + auto &alpha_reg = zmm[alpha_load_reg]; co2 = co1 + ldc; if (!is_alpha1) alpha_reg = pload1<vec>(alpha); @@ -542,8 +542,8 @@ // compute template <int um, int um_vecs, int idx, int uk, bool fetch_x, bool ktail> - EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> compute(const Scalar* ao, const Scalar* bo, int& fetchA_idx, - int& fetchB_idx, vec& b_reg) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> compute(const Scalar *ao, const Scalar *bo, int &fetchA_idx, + int &fetchB_idx, vec &b_reg) { EIGEN_UNUSED_VARIABLE(ao); EIGEN_UNUSED_VARIABLE(bo); EIGEN_UNUSED_VARIABLE(fetchA_idx); @@ -552,11 +552,11 @@ } template <int um, int um_vecs, int idx, int uk, bool fetch_x, bool ktail> - EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> compute(const Scalar* ao, const Scalar* bo, int& fetchA_idx, - int& fetchB_idx, vec& b_reg) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> compute(const Scalar *ao, const Scalar *bo, int &fetchA_idx, + int &fetchB_idx, vec &b_reg) { if (um < um_vecs) { - auto& c_reg = zmm[c_regs[um + idx * 3]]; - auto& a_reg = zmm[a_regs[um + (uk % 2) * 3]]; + auto &c_reg = zmm[c_regs[um + idx * 3]]; + auto &a_reg = zmm[a_regs[um + (uk % 2) * 3]]; vfmadd(c_reg, a_reg, b_reg); @@ -578,25 +578,25 @@ // load_a template <int um, int um_vecs, int uk, int nelems, bool ktail> - EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> load_a(const Scalar* ao) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> load_a(const Scalar *ao) { EIGEN_UNUSED_VARIABLE(ao); } template <int um, int um_vecs, int uk, int nelems, bool ktail> - EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> load_a(const Scalar* ao) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> load_a(const Scalar *ao) { if (um < um_vecs) { - auto& a_reg = zmm[a_regs[um + (uk % 2) * 3]]; - const Scalar* a_addr = ao + nelems * (1 + !ktail * !use_less_a_regs + uk) + nelems_in_cache_line * um - a_shift; + auto &a_reg = zmm[a_regs[um + (uk % 2) * 3]]; + const Scalar *a_addr = ao + nelems * (1 + !ktail * !use_less_a_regs + uk) + nelems_in_cache_line * um - a_shift; a_load<nelems>(a_reg, a_addr); load_a<um + 1, um_vecs, uk, nelems, ktail>(ao); } } template <int uk, int pow, int count, int um_vecs, int b_unroll, bool ktail, bool fetch_x, bool c_fetch> - EIGEN_ALWAYS_INLINE std::enable_if_t<(count > (pow + 1) / 2)> innerkernel_1pow(const Scalar*& aa, - const Scalar* const& ao, - const Scalar* const& bo, Scalar*& co2, - int& fetchA_idx, int& fetchB_idx) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(count > (pow + 1) / 2)> innerkernel_1pow(const Scalar *&aa, + const Scalar *const &ao, + const Scalar *const &bo, Scalar *&co2, + int &fetchA_idx, int &fetchB_idx) { EIGEN_UNUSED_VARIABLE(aa); EIGEN_UNUSED_VARIABLE(ao); EIGEN_UNUSED_VARIABLE(bo); @@ -606,14 +606,14 @@ } template <int uk, int pow, int count, int um_vecs, int b_unroll, bool ktail, bool fetch_x, bool c_fetch> - EIGEN_ALWAYS_INLINE std::enable_if_t<(count <= (pow + 1) / 2)> innerkernel_1pow(const Scalar*& aa, - const Scalar* const& ao, - const Scalar* const& bo, Scalar*& co2, - int& fetchA_idx, int& fetchB_idx) { + EIGEN_ALWAYS_INLINE std::enable_if_t<(count <= (pow + 1) / 2)> innerkernel_1pow(const Scalar *&aa, + const Scalar *const &ao, + const Scalar *const &bo, Scalar *&co2, + int &fetchA_idx, int &fetchB_idx) { const int idx = (pow / 2) + count; if (count < (pow + 1) / 2) { - auto& b_reg = zmm[b_regs[idx % 2]]; + auto &b_reg = zmm[b_regs[idx % 2]]; if (fetch_x && uk == 3 && idx == 0) prefetch_x(aa); if (fetch_x && uk == 3 && idx == 4) aa += 8; @@ -621,7 +621,7 @@ if (b_unroll >= pow) { compute<0, um_vecs, idx, uk, fetch_x, ktail>(ao, bo, fetchA_idx, fetchB_idx, b_reg); - const Scalar* b_addr = bo + b_unroll * uk + idx + 1 + (b_unroll > 1) * !use_less_b_regs - b_shift; + const Scalar *b_addr = bo + b_unroll * uk + idx + 1 + (b_unroll > 1) * !use_less_b_regs - b_shift; b_load(b_reg, b_addr); } @@ -643,8 +643,8 @@ template <int uk, int max_b_unroll, int a_unroll, int b_unroll, bool ktail, bool fetch_x, bool c_fetch, bool no_a_preload = false> - EIGEN_ALWAYS_INLINE void innerkernel_1uk(const Scalar*& aa, const Scalar* const& ao, const Scalar* const& bo, - Scalar*& co2, int& fetchA_idx, int& fetchB_idx) { + EIGEN_ALWAYS_INLINE void innerkernel_1uk(const Scalar *&aa, const Scalar *const &ao, const Scalar *const &bo, + Scalar *&co2, int &fetchA_idx, int &fetchB_idx) { const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); if (max_b_unroll >= 1) @@ -701,7 +701,7 @@ template <int a_unroll, int b_unroll, int k_factor, int max_b_unroll, int max_k_factor, bool c_fetch, bool no_a_preload = false> - EIGEN_ALWAYS_INLINE void innerkernel(const Scalar*& aa, const Scalar*& ao, const Scalar*& bo, Scalar*& co2) { + EIGEN_ALWAYS_INLINE void innerkernel(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co2) { int fetchA_idx = 0; int fetchB_idx = 0; @@ -731,7 +731,7 @@ } template <int a_unroll, int b_unroll, int max_b_unroll> - EIGEN_ALWAYS_INLINE void kloop(const Scalar*& aa, const Scalar*& ao, const Scalar*& bo, Scalar*& co1, Scalar*& co2) { + EIGEN_ALWAYS_INLINE void kloop(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); if (!use_less_a_regs && k > 1) a_loads<0, 2, 0, um_vecs, a_unroll>(ao); @@ -795,7 +795,7 @@ } template <int a_unroll, int b_unroll, int max_b_unroll> - EIGEN_ALWAYS_INLINE void nloop(const Scalar*& aa, const Scalar*& ao, const Scalar*& bo, Scalar*& co1, Scalar*& co2) { + EIGEN_ALWAYS_INLINE void nloop(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { // Set A matrix pointer. ao = a + a_off * a_unroll; @@ -812,9 +812,9 @@ } template <int a_unroll, int max_a_unroll, int max_b_unroll> - EIGEN_ALWAYS_INLINE void mloop(const Scalar*& ao, const Scalar*& bo, Scalar*& co1, Scalar*& co2) { + EIGEN_ALWAYS_INLINE void mloop(const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { // Set prefetch A pointers. - const Scalar* aa = a + a_unroll * a_stride; + const Scalar *aa = a + a_unroll * a_stride; // Set C matrix pointers. co1 = c; @@ -856,10 +856,10 @@ a -= -a_shift; b -= -b_shift; - const Scalar* ao = nullptr; - const Scalar* bo = nullptr; - Scalar* co1 = nullptr; - Scalar* co2 = nullptr; + const Scalar *ao = nullptr; + const Scalar *bo = nullptr; + Scalar *co1 = nullptr; + Scalar *co2 = nullptr; // Main m-loop. for (; m >= max_a_unroll; m -= max_a_unroll) mloop<max_a_unroll, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); @@ -883,8 +883,8 @@ } } - gemm_class(Index m_, Index n_, Index k_, Index ldc_, Index inc_, const Scalar* alpha_, const Scalar* a_, - const Scalar* b_, Scalar* c_, bool is_alpha1_, bool is_beta0_, Index a_stride_, Index b_stride_, + gemm_class(Index m_, Index n_, Index k_, Index ldc_, Index inc_, const Scalar *alpha_, const Scalar *a_, + const Scalar *b_, Scalar *c_, bool is_alpha1_, bool is_beta0_, Index a_stride_, Index b_stride_, Index a_off_, Index b_off_) : m(m_), n(n_), @@ -937,8 +937,8 @@ // max_a_unroll: 24, 16, 8, 4, 2, 1 // max_b_unroll: 8, 4, 2, 1 template <typename Scalar, int max_a_unroll, int max_b_unroll, bool is_alpha1, bool is_beta0, bool is_unit_inc> -EIGEN_DONT_INLINE void gemm_kern_avx512(Index m, Index n, Index k, Scalar* alpha, const Scalar* a, const Scalar* b, - Scalar* c, Index ldc, Index inc = 1, Index a_stride = -1, Index b_stride = -1, +EIGEN_DONT_INLINE void gemm_kern_avx512(Index m, Index n, Index k, Scalar *alpha, const Scalar *a, const Scalar *b, + Scalar *c, Index ldc, Index inc = 1, Index a_stride = -1, Index b_stride = -1, Index a_off = 0, Index b_off = 0) { if (a_stride == -1) a_stride = k; if (b_stride == -1) b_stride = k; @@ -972,13 +972,13 @@ typedef typename packet_traits<Scalar>::type Packet; typedef typename DataMapper::LinearMapper LinearMapper; enum { PacketSize = packet_traits<Scalar>::size }; - EIGEN_DONT_INLINE void operator()(Scalar* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0, + EIGEN_DONT_INLINE void operator()(Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride = 0, Index offset = 0); }; template <typename Scalar, typename Index, typename DataMapper, bool Conjugate, bool PanelMode> EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, 8, ColMajor, Conjugate, PanelMode>::operator()( - Scalar* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) { + Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride, Index offset) { constexpr int nr = 8; EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK RHS COLMAJOR"); EIGEN_UNUSED_VARIABLE(stride); @@ -1106,7 +1106,7 @@ HalfPacketSize = unpacket_traits<HalfPacket>::size, QuarterPacketSize = unpacket_traits<QuarterPacket>::size }; - EIGEN_DONT_INLINE void operator()(Scalar* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0, + EIGEN_DONT_INLINE void operator()(Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride = 0, Index offset = 0) { constexpr int nr = 8; EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK RHS ROWMAJOR"); @@ -1205,32 +1205,33 @@ template <typename Scalar, typename Index, typename DataMapper, int mr, bool ConjugateLhs, bool ConjugateRhs> struct gebp_kernel<Scalar, Scalar, Index, DataMapper, mr, 8, ConjugateLhs, ConjugateRhs> { - EIGEN_ALWAYS_INLINE void operator()(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, + EIGEN_ALWAYS_INLINE void operator()(const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA = -1, Index strideB = -1, Index offsetA = 0, Index offsetB = 0); }; template <typename Scalar, typename Index, typename DataMapper, int mr, bool ConjugateLhs, bool ConjugateRhs> EIGEN_ALWAYS_INLINE void gebp_kernel<Scalar, Scalar, Index, DataMapper, mr, 8, ConjugateLhs, ConjugateRhs>::operator()( - const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, + const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { if (res.incr() == 1) { if (alpha == 1) { - gemm_kern_avx512<Scalar, mr, 8, true, false, true>(rows, cols, depth, &alpha, blockA, blockB, (Scalar*)res.data(), - res.stride(), res.incr(), strideA, strideB, offsetA, offsetB); + gemm_kern_avx512<Scalar, mr, 8, true, false, true>(rows, cols, depth, &alpha, blockA, blockB, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, + strideB, offsetA, offsetB); } else { gemm_kern_avx512<Scalar, mr, 8, false, false, true>(rows, cols, depth, &alpha, blockA, blockB, - (Scalar*)res.data(), res.stride(), res.incr(), strideA, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, strideB, offsetA, offsetB); } } else { if (alpha == 1) { gemm_kern_avx512<Scalar, mr, 8, true, false, false>(rows, cols, depth, &alpha, blockA, blockB, - (Scalar*)res.data(), res.stride(), res.incr(), strideA, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, strideB, offsetA, offsetB); } else { gemm_kern_avx512<Scalar, mr, 8, false, false, false>(rows, cols, depth, &alpha, blockA, blockB, - (Scalar*)res.data(), res.stride(), res.incr(), strideA, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, strideB, offsetA, offsetB); } }
diff --git a/Eigen/src/Core/arch/AVX512/PacketMath.h b/Eigen/src/Core/arch/AVX512/PacketMath.h index ddc766b..c8017e4 100644 --- a/Eigen/src/Core/arch/AVX512/PacketMath.h +++ b/Eigen/src/Core/arch/AVX512/PacketMath.h
@@ -100,6 +100,7 @@ HasCos = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, + HasBlend = 0 }; }; #endif @@ -117,6 +118,7 @@ HasMin = 1, HasMax = 1, HasConj = 1, + HasBlend = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasACos = 1, @@ -149,6 +151,7 @@ Vectorizable = 1, AlignedOnScalar = 1, size = 8, + HasBlend = 1, HasSqrt = 1, HasRsqrt = 1, HasCbrt = 1, @@ -173,7 +176,7 @@ struct packet_traits<int> : default_packet_traits { typedef Packet16i type; typedef Packet8i half; - enum { Vectorizable = 1, AlignedOnScalar = 1, HasCmp = 1, HasDiv = 1, size = 16 }; + enum { Vectorizable = 1, AlignedOnScalar = 1, HasBlend = 0, HasCmp = 1, HasDiv = 1, size = 16 }; }; template <> @@ -2509,6 +2512,7 @@ Vectorizable = 1, AlignedOnScalar = 1, size = 16, + HasBlend = 0, HasInsert = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH,
diff --git a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h index e6b8d99..b8fa208 100644 --- a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h +++ b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h
@@ -59,6 +59,7 @@ HasCos = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = 0, // EIGEN_FAST_MATH, + HasBlend = 0 }; };
diff --git a/Eigen/src/Core/arch/AVX512/TrsmKernel.h b/Eigen/src/Core/arch/AVX512/TrsmKernel.h index 11c0bd7..cb72152 100644 --- a/Eigen/src/Core/arch/AVX512/TrsmKernel.h +++ b/Eigen/src/Core/arch/AVX512/TrsmKernel.h
@@ -44,7 +44,7 @@ namespace Eigen { namespace internal { -#if (EIGEN_USE_AVX512_TRSM_KERNELS) +#if (EIGEN_USE_AVX512_TRSM_KERNELS) && (EIGEN_COMP_CLANG != 0) #define EIGEN_AVX_MAX_NUM_ACC (int64_t(24)) #define EIGEN_AVX_MAX_NUM_ROW (int64_t(8)) // Denoted L in code. @@ -60,8 +60,6 @@ // Note: this depends on macros and typedefs above. #include "TrsmUnrolls.inc" -#if (EIGEN_COMP_CLANG != 0) - /** * For smaller problem sizes, and certain compilers, using the optimized kernels trsmKernelL/R directly * is faster than the packed versions in TriangularSolverMatrix.h. @@ -121,7 +119,7 @@ * Used by gemmKernel for the case A/B row-major and C col-major. */ template <typename Scalar, typename vec, int64_t unrollM, int64_t unrollN, bool remM, bool remN> -EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS>& zmm, Scalar* C_arr, +EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> &zmm, Scalar *C_arr, int64_t LDC, int64_t remM_ = 0, int64_t remN_ = 0) { EIGEN_UNUSED_VARIABLE(remN_); EIGEN_UNUSED_VARIABLE(remM_); @@ -221,7 +219,7 @@ * handleKRem: Handle arbitrary K? This is not needed for trsm. */ template <typename Scalar, bool isARowMajor, bool isCRowMajor, bool isAdd, bool handleKRem> -void gemmKernel(Scalar* A_arr, Scalar* B_arr, Scalar* C_arr, int64_t M, int64_t N, int64_t K, int64_t LDA, int64_t LDB, +void gemmKernel(Scalar *A_arr, Scalar *B_arr, Scalar *C_arr, int64_t M, int64_t N, int64_t K, int64_t LDA, int64_t LDB, int64_t LDC) { using urolls = unrolls::gemm<Scalar, isAdd>; constexpr int64_t U3 = urolls::PacketSize * 3; @@ -264,8 +262,8 @@ } } if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<3, 4>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -294,8 +292,8 @@ i += 4; } if (M - i >= 2) { - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<3, 2>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -324,8 +322,8 @@ i += 2; } if (M - i > 0) { - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<3, 1>(zmm); { @@ -387,8 +385,8 @@ } } if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<2, 4>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -417,8 +415,8 @@ i += 4; } if (M - i >= 2) { - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<2, 2>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -447,8 +445,8 @@ i += 2; } if (M - i > 0) { - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<2, 1>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -508,8 +506,8 @@ } } if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 4>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -538,8 +536,8 @@ i += 4; } if (M - i >= 2) { - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 2>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -568,8 +566,8 @@ i += 2; } if (M - i > 0) { - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 1>(zmm); { @@ -603,8 +601,8 @@ constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 1; int64_t i = 0; for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, EIGEN_AVX_MAX_NUM_ROW>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -632,8 +630,8 @@ } } if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 4>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -662,8 +660,8 @@ i += 4; } if (M - i >= 2) { - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 2>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -692,8 +690,8 @@ i += 2; } if (M - i > 0) { - Scalar* A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; - Scalar* B_t = &B_arr[0 * LDB + j]; + Scalar *A_t = &A_arr[idA<isARowMajor>(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; PacketBlock<vec, EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS> zmm; urolls::template setzero<1, 1>(zmm); for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { @@ -732,7 +730,7 @@ * The B matrix (RHS) is assumed to be row-major */ template <typename Scalar, typename vec, int64_t unrollM, bool isARowMajor, bool isFWDSolve, bool isUnitDiag> -EIGEN_ALWAYS_INLINE void triSolveKernel(Scalar* A_arr, Scalar* B_arr, int64_t K, int64_t LDA, int64_t LDB) { +EIGEN_ALWAYS_INLINE void triSolveKernel(Scalar *A_arr, Scalar *B_arr, int64_t K, int64_t LDA, int64_t LDB) { static_assert(unrollM <= EIGEN_AVX_MAX_NUM_ROW, "unrollM should be equal to EIGEN_AVX_MAX_NUM_ROW"); using urolls = unrolls::trsm<Scalar>; constexpr int64_t U3 = urolls::PacketSize * 3; @@ -782,7 +780,7 @@ * The B matrix (RHS) is assumed to be row-major */ template <typename Scalar, bool isARowMajor, bool isFWDSolve, bool isUnitDiag> -void triSolveKernelLxK(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t K, int64_t LDA, int64_t LDB) { +void triSolveKernelLxK(Scalar *A_arr, Scalar *B_arr, int64_t M, int64_t K, int64_t LDA, int64_t LDB) { // Note: this assumes EIGEN_AVX_MAX_NUM_ROW = 8. Unrolls should be adjusted // accordingly if EIGEN_AVX_MAX_NUM_ROW is smaller. using vec = typename std::conditional<std::is_same<Scalar, float>::value, vecFullFloat, vecFullDouble>::type; @@ -813,7 +811,7 @@ * */ template <typename Scalar, bool toTemp = true, bool remM = false> -EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar* B_arr, int64_t LDB, int64_t K, Scalar* B_temp, int64_t LDB_, +EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar *B_arr, int64_t LDB, int64_t K, Scalar *B_temp, int64_t LDB_, int64_t remM_ = 0) { EIGEN_UNUSED_VARIABLE(remM_); using urolls = unrolls::transB<Scalar>; @@ -900,7 +898,7 @@ */ template <typename Scalar, bool isARowMajor = true, bool isBRowMajor = true, bool isFWDSolve = true, bool isUnitDiag = false> -void triSolve(Scalar* A_arr, Scalar* B_arr, int64_t M, int64_t numRHS, int64_t LDA, int64_t LDB) { +void triSolve(Scalar *A_arr, Scalar *B_arr, int64_t M, int64_t numRHS, int64_t LDA, int64_t LDB) { constexpr int64_t psize = packet_traits<Scalar>::size; /** * The values for kB, numM were determined experimentally. @@ -919,7 +917,7 @@ constexpr int64_t numM = 8 * EIGEN_AVX_MAX_NUM_ROW; int64_t sizeBTemp = 0; - Scalar* B_temp = NULL; + Scalar *B_temp = NULL; EIGEN_IF_CONSTEXPR(!isBRowMajor) { /** * If B is col-major, we copy it to a fixed-size temporary array of size at most ~numM*kB and @@ -929,7 +927,7 @@ sizeBTemp = (((std::min(kB, numRHS) + psize - 1) / psize + 4) * psize) * numM; } - EIGEN_IF_CONSTEXPR(!isBRowMajor) B_temp = (Scalar*)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64); + EIGEN_IF_CONSTEXPR(!isBRowMajor) B_temp = (Scalar *)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64); for (int64_t k = 0; k < numRHS; k += kB) { int64_t bK = numRHS - k > kB ? kB : numRHS - k; @@ -1063,6 +1061,7 @@ } // Template specializations of trsmKernelL/R for float/double and inner strides of 1. +#if (EIGEN_USE_AVX512_TRSM_KERNELS) #if (EIGEN_USE_AVX512_TRSM_R_KERNELS) template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder, int OtherInnerStride, bool Specialized> @@ -1070,19 +1069,19 @@ template <typename Index, int Mode, int TriStorageOrder> struct trsmKernelR<float, Index, Mode, false, TriStorageOrder, 1, true> { - static void kernel(Index size, Index otherSize, const float* _tri, Index triStride, float* _other, Index otherIncr, + static void kernel(Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, Index otherStride); }; template <typename Index, int Mode, int TriStorageOrder> struct trsmKernelR<double, Index, Mode, false, TriStorageOrder, 1, true> { - static void kernel(Index size, Index otherSize, const double* _tri, Index triStride, double* _other, Index otherIncr, + static void kernel(Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, Index otherStride); }; template <typename Index, int Mode, int TriStorageOrder> EIGEN_DONT_INLINE void trsmKernelR<float, Index, Mode, false, TriStorageOrder, 1, true>::kernel( - Index size, Index otherSize, const float* _tri, Index triStride, float* _other, Index otherIncr, + Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, Index otherStride) { EIGEN_UNUSED_VARIABLE(otherIncr); #ifdef EIGEN_RUNTIME_NO_MALLOC @@ -1093,12 +1092,12 @@ } #endif triSolve<float, TriStorageOrder != RowMajor, true, (Mode & Lower) != Lower, (Mode & UnitDiag) != 0>( - const_cast<float*>(_tri), _other, size, otherSize, triStride, otherStride); + const_cast<float *>(_tri), _other, size, otherSize, triStride, otherStride); } template <typename Index, int Mode, int TriStorageOrder> EIGEN_DONT_INLINE void trsmKernelR<double, Index, Mode, false, TriStorageOrder, 1, true>::kernel( - Index size, Index otherSize, const double* _tri, Index triStride, double* _other, Index otherIncr, + Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, Index otherStride) { EIGEN_UNUSED_VARIABLE(otherIncr); #ifdef EIGEN_RUNTIME_NO_MALLOC @@ -1109,7 +1108,7 @@ } #endif triSolve<double, TriStorageOrder != RowMajor, true, (Mode & Lower) != Lower, (Mode & UnitDiag) != 0>( - const_cast<double*>(_tri), _other, size, otherSize, triStride, otherStride); + const_cast<double *>(_tri), _other, size, otherSize, triStride, otherStride); } #endif // (EIGEN_USE_AVX512_TRSM_R_KERNELS) @@ -1121,19 +1120,19 @@ template <typename Index, int Mode, int TriStorageOrder> struct trsmKernelL<float, Index, Mode, false, TriStorageOrder, 1, true> { - static void kernel(Index size, Index otherSize, const float* _tri, Index triStride, float* _other, Index otherIncr, + static void kernel(Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, Index otherStride); }; template <typename Index, int Mode, int TriStorageOrder> struct trsmKernelL<double, Index, Mode, false, TriStorageOrder, 1, true> { - static void kernel(Index size, Index otherSize, const double* _tri, Index triStride, double* _other, Index otherIncr, + static void kernel(Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, Index otherStride); }; template <typename Index, int Mode, int TriStorageOrder> EIGEN_DONT_INLINE void trsmKernelL<float, Index, Mode, false, TriStorageOrder, 1, true>::kernel( - Index size, Index otherSize, const float* _tri, Index triStride, float* _other, Index otherIncr, + Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, Index otherStride) { EIGEN_UNUSED_VARIABLE(otherIncr); #ifdef EIGEN_RUNTIME_NO_MALLOC @@ -1144,12 +1143,12 @@ } #endif triSolve<float, TriStorageOrder == RowMajor, false, (Mode & Lower) == Lower, (Mode & UnitDiag) != 0>( - const_cast<float*>(_tri), _other, size, otherSize, triStride, otherStride); + const_cast<float *>(_tri), _other, size, otherSize, triStride, otherStride); } template <typename Index, int Mode, int TriStorageOrder> EIGEN_DONT_INLINE void trsmKernelL<double, Index, Mode, false, TriStorageOrder, 1, true>::kernel( - Index size, Index otherSize, const double* _tri, Index triStride, double* _other, Index otherIncr, + Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, Index otherStride) { EIGEN_UNUSED_VARIABLE(otherIncr); #ifdef EIGEN_RUNTIME_NO_MALLOC @@ -1160,12 +1159,10 @@ } #endif triSolve<double, TriStorageOrder == RowMajor, false, (Mode & Lower) == Lower, (Mode & UnitDiag) != 0>( - const_cast<double*>(_tri), _other, size, otherSize, triStride, otherStride); + const_cast<double *>(_tri), _other, size, otherSize, triStride, otherStride); } #endif // EIGEN_USE_AVX512_TRSM_L_KERNELS - #endif // EIGEN_USE_AVX512_TRSM_KERNELS - } // namespace internal } // namespace Eigen #endif // EIGEN_CORE_ARCH_AVX512_TRSM_KERNEL_H
diff --git a/Eigen/src/Core/arch/GPU/PacketMath.h b/Eigen/src/Core/arch/GPU/PacketMath.h index 7d062f3..328b1b9 100644 --- a/Eigen/src/Core/arch/GPU/PacketMath.h +++ b/Eigen/src/Core/arch/GPU/PacketMath.h
@@ -84,6 +84,7 @@ HasIGammac = 1, HasBetaInc = 1, + HasBlend = 0, HasFloor = 1, HasCmp = EIGEN_HAS_GPU_DEVICE_FUNCTIONS }; @@ -116,6 +117,7 @@ HasGammaSampleDerAlpha = 1, HasIGammac = 1, HasBetaInc = 1, + HasBlend = 0, }; };
diff --git a/Eigen/src/Core/arch/HVX/PacketMath.h b/Eigen/src/Core/arch/HVX/PacketMath.h index db543e0..d9f5b07 100644 --- a/Eigen/src/Core/arch/HVX/PacketMath.h +++ b/Eigen/src/Core/arch/HVX/PacketMath.h
@@ -157,7 +157,10 @@ HasMax = 1, HasConj = 0, HasSetLinear = 0, + HasBlend = 0, + HasDiv = 0, + HasSin = 0, HasCos = 0, HasACos = 0,
diff --git a/Eigen/src/Core/arch/SSE/PacketMath.h b/Eigen/src/Core/arch/SSE/PacketMath.h index 1ea23b0..b8b1d9a 100644 --- a/Eigen/src/Core/arch/SSE/PacketMath.h +++ b/Eigen/src/Core/arch/SSE/PacketMath.h
@@ -200,6 +200,7 @@ HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, HasErfc = EIGEN_FAST_MATH, + HasBlend = 1, HasSign = 0 // The manually vectorized version is slightly slower for SSE. }; }; @@ -229,6 +230,7 @@ HasCbrt = 1, HasATan = 1, HasATanh = 1, + HasBlend = 1 }; }; template <> @@ -243,6 +245,7 @@ HasCmp = 1, HasDiv = 1, HasShift = 1, + HasBlend = 1 }; }; template <> @@ -254,9 +257,11 @@ AlignedOnScalar = 1, size = 4, + HasDiv = 0, HasNegate = 0, HasCmp = 1, HasShift = 1, + HasBlend = 1 }; }; template <> @@ -268,8 +273,10 @@ AlignedOnScalar = 1, size = 2, + HasDiv = 0, HasCmp = 1, HasShift = 1, + HasBlend = 1 }; }; #endif
diff --git a/Eigen/src/Core/arch/SYCL/InteropHeaders.h b/Eigen/src/Core/arch/SYCL/InteropHeaders.h index 0bdf825..578e0f3 100644 --- a/Eigen/src/Core/arch/SYCL/InteropHeaders.h +++ b/Eigen/src/Core/arch/SYCL/InteropHeaders.h
@@ -30,7 +30,7 @@ namespace internal { -template <int lengths> +template <int has_blend, int lengths> struct sycl_packet_traits : default_packet_traits { enum { Vectorizable = 1, @@ -60,6 +60,7 @@ HasIGamma = 0, HasIGammac = 0, HasBetaInc = 0, + HasBlend = has_blend, // This flag is used to indicate whether packet comparison is supported. // pcmp_eq, pcmp_lt and pcmp_le should be defined for it to be true. HasCmp = 1, @@ -77,19 +78,19 @@ }; #ifdef SYCL_DEVICE_ONLY -#define SYCL_PACKET_TRAITS(packet_type, unpacket_type, lengths) \ - template <> \ - struct packet_traits<unpacket_type> : sycl_packet_traits<lengths> { \ - typedef packet_type type; \ - typedef packet_type half; \ +#define SYCL_PACKET_TRAITS(packet_type, has_blend, unpacket_type, lengths) \ + template <> \ + struct packet_traits<unpacket_type> : sycl_packet_traits<has_blend, lengths> { \ + typedef packet_type type; \ + typedef packet_type half; \ }; -SYCL_PACKET_TRAITS(cl::sycl::cl_half8, Eigen::half, 8) -SYCL_PACKET_TRAITS(cl::sycl::cl_half8, const Eigen::half, 8) -SYCL_PACKET_TRAITS(cl::sycl::cl_float4, float, 4) -SYCL_PACKET_TRAITS(cl::sycl::cl_float4, const float, 4) -SYCL_PACKET_TRAITS(cl::sycl::cl_double2, double, 2) -SYCL_PACKET_TRAITS(cl::sycl::cl_double2, const double, 2) +SYCL_PACKET_TRAITS(cl::sycl::cl_half8, 1, Eigen::half, 8) +SYCL_PACKET_TRAITS(cl::sycl::cl_half8, 1, const Eigen::half, 8) +SYCL_PACKET_TRAITS(cl::sycl::cl_float4, 1, float, 4) +SYCL_PACKET_TRAITS(cl::sycl::cl_float4, 1, const float, 4) +SYCL_PACKET_TRAITS(cl::sycl::cl_double2, 0, double, 2) +SYCL_PACKET_TRAITS(cl::sycl::cl_double2, 0, const double, 2) #undef SYCL_PACKET_TRAITS // Make sure this is only available when targeting a GPU: we don't want to @@ -134,14 +135,14 @@ struct PacketWrapper { typedef typename ::Eigen::internal::unpacket_traits<PacketReturnType>::type Scalar; template <typename Index> - EIGEN_DEVICE_FUNC static Scalar scalarize(Index, PacketReturnType&) { + EIGEN_DEVICE_FUNC static Scalar scalarize(Index, PacketReturnType &) { eigen_assert(false && "THERE IS NO PACKETIZE VERSION FOR THE CHOSEN TYPE"); abort(); } EIGEN_DEVICE_FUNC static PacketReturnType convert_to_packet_type(Scalar in, Scalar) { return ::Eigen::internal::template plset<PacketReturnType>(in); } - EIGEN_DEVICE_FUNC static void set_packet(PacketReturnType, Scalar*) { + EIGEN_DEVICE_FUNC static void set_packet(PacketReturnType, Scalar *) { eigen_assert(false && "THERE IS NO PACKETIZE VERSION FOR THE CHOSEN TYPE"); abort(); } @@ -152,7 +153,7 @@ struct PacketWrapper<PacketReturnType, 4> { typedef typename ::Eigen::internal::unpacket_traits<PacketReturnType>::type Scalar; template <typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index index, PacketReturnType& in) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index index, PacketReturnType &in) { switch (index) { case 0: return in.x(); @@ -173,7 +174,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType convert_to_packet_type(Scalar in, Scalar other) { return PacketReturnType(in, other, other, other); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType& lhs, Scalar* rhs) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType &lhs, Scalar *rhs) { lhs = PacketReturnType(rhs[0], rhs[1], rhs[2], rhs[3]); } }; @@ -182,20 +183,20 @@ struct PacketWrapper<PacketReturnType, 1> { typedef typename ::Eigen::internal::unpacket_traits<PacketReturnType>::type Scalar; template <typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index, PacketReturnType& in) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index, PacketReturnType &in) { return in; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType convert_to_packet_type(Scalar in, Scalar) { return PacketReturnType(in); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType& lhs, Scalar* rhs) { lhs = rhs[0]; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType &lhs, Scalar *rhs) { lhs = rhs[0]; } }; template <typename PacketReturnType> struct PacketWrapper<PacketReturnType, 2> { typedef typename ::Eigen::internal::unpacket_traits<PacketReturnType>::type Scalar; template <typename Index> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index index, PacketReturnType& in) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static Scalar scalarize(Index index, PacketReturnType &in) { switch (index) { case 0: return in.x(); @@ -212,7 +213,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType convert_to_packet_type(Scalar in, Scalar other) { return PacketReturnType(in, other); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType& lhs, Scalar* rhs) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void set_packet(PacketReturnType &lhs, Scalar *rhs) { lhs = PacketReturnType(rhs[0], rhs[1]); } };
diff --git a/Eigen/src/Core/arch/clang/PacketMath.h b/Eigen/src/Core/arch/clang/PacketMath.h index 55ddc1c..359f1e4 100644 --- a/Eigen/src/Core/arch/clang/PacketMath.h +++ b/Eigen/src/Core/arch/clang/PacketMath.h
@@ -31,9 +31,13 @@ using Packet8l = detail::VectorType<int64_t, 8>; // --- packet_traits specializations --- -struct generic_float_packet_traits : default_packet_traits { +template <> +struct packet_traits<float> : default_packet_traits { + using type = Packet16f; + using half = Packet16f; enum { Vectorizable = 1, + size = 16, AlignedOnScalar = 1, HasAdd = 1, HasSub = 1, @@ -42,8 +46,7 @@ HasNegate = 1, HasAbs = 1, HasRound = 1, - HasMin = 1, - HasMax = 1, + HasMinMax = 1, HasCmp = 1, HasSet1 = 1, HasCast = 1, @@ -77,24 +80,12 @@ }; template <> -struct packet_traits<float> : generic_float_packet_traits { - using type = Packet16f; - using half = Packet16f; - enum { - size = 16, - }; -}; - -template <> -struct packet_traits<double> : generic_float_packet_traits { +struct packet_traits<double> : default_packet_traits { using type = Packet8d; using half = Packet8d; - enum { size = 8, HasACos = 0, HasASin = 0 }; -}; - -struct generic_integer_packet_traits : default_packet_traits { enum { Vectorizable = 1, + size = 8, AlignedOnScalar = 1, HasAdd = 1, HasSub = 1, @@ -102,8 +93,55 @@ HasDiv = 1, HasNegate = 1, HasAbs = 1, - HasMin = 1, - HasMax = 1, + HasRound = 1, + HasMinMax = 1, + HasCmp = 1, + HasSet1 = 1, + HasCast = 1, + HasBitwise = 1, + HasRedux = 1, + HasSign = 1, + HasArg = 0, + HasConj = 1, + // Math functions + HasReciprocal = 1, + HasSin = 1, + HasCos = 1, + HasACos = 0, + HasASin = 0, + HasATan = 1, + HasATanh = 1, + HasLog = 1, + HasLog1p = 1, + HasExpm1 = 1, + HasExp = 1, + HasPow = 1, + HasNdtri = 1, + HasBessel = 1, + HasSqrt = 1, + HasRsqrt = 1, + HasCbrt = 1, + HasTanh = 1, + HasErf = 1, + HasErfc = 1 + }; +}; + +template <> +struct packet_traits<int32_t> : default_packet_traits { + using type = Packet16i; + using half = Packet16i; + enum { + Vectorizable = 1, + size = 16, + AlignedOnScalar = 1, + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasAbs = 1, + HasMinMax = 1, HasCmp = 1, HasSet1 = 1, HasCast = 1, @@ -124,63 +162,88 @@ }; template <> -struct packet_traits<int32_t> : generic_integer_packet_traits { - using type = Packet16i; - using half = Packet16i; - enum { - size = 16, - }; -}; - -template <> -struct packet_traits<int64_t> : generic_integer_packet_traits { +struct packet_traits<int64_t> : default_packet_traits { using type = Packet8l; using half = Packet8l; enum { + Vectorizable = 1, size = 8, + AlignedOnScalar = 1, + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasAbs = 1, + HasMinMax = 1, + HasCmp = 1, + HasSet1 = 1, + HasCast = 1, + HasBitwise = 1, + HasRedux = 1, + // Set remaining to 0 + HasRound = 1, + HasSqrt = 0, + HasRsqrt = 0, + HasReciprocal = 0, + HasArg = 0, + HasConj = 1, + HasExp = 0, + HasLog = 0, + HasSin = 0, + HasCos = 0, }; }; // --- unpacket_traits specializations --- -struct generic_unpacket_traits : default_unpacket_traits { - enum { - alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, - vectorizable = true, - }; -}; - template <> -struct unpacket_traits<Packet16f> : generic_unpacket_traits { +struct unpacket_traits<Packet16f> { using type = float; using half = Packet16f; using integer_packet = Packet16i; enum { size = 16, + alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, + vectorizable = true, + masked_load_available = false, + masked_store_available = false }; }; template <> -struct unpacket_traits<Packet8d> : generic_unpacket_traits { +struct unpacket_traits<Packet8d> { using type = double; using half = Packet8d; using integer_packet = Packet8l; enum { size = 8, + alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, + vectorizable = true, + masked_load_available = false, + masked_store_available = false }; }; template <> -struct unpacket_traits<Packet16i> : generic_unpacket_traits { +struct unpacket_traits<Packet16i> { using type = int32_t; using half = Packet16i; enum { size = 16, + alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, + vectorizable = true, + masked_load_available = false, + masked_store_available = false }; }; template <> -struct unpacket_traits<Packet8l> : generic_unpacket_traits { +struct unpacket_traits<Packet8l> { using type = int64_t; using half = Packet8l; enum { size = 8, + alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES, + vectorizable = true, + masked_load_available = false, + masked_store_available = false }; }; @@ -395,8 +458,8 @@ #undef EIGEN_CLANG_PACKET_BITWISE_FLOAT // --- Min/Max operations --- -#if EIGEN_HAS_BUILTIN(__builtin_elementwise_min) && EIGEN_HAS_BUILTIN(__builtin_elementwise_max) && \ - EIGEN_HAS_BUILTIN(__builtin_elementwise_abs) +#if __has_builtin(__builtin_elementwise_min) && __has_builtin(__builtin_elementwise_max) && \ + __has_builtin(__builtin_elementwise_abs) #define EIGEN_CLANG_PACKET_ELEMENTWISE(PACKET_TYPE) \ template <> \ EIGEN_STRONG_INLINE PACKET_TYPE pmin<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \ @@ -431,7 +494,7 @@ template <> \ EIGEN_STRONG_INLINE PACKET_TYPE pselect<PACKET_TYPE>(const PACKET_TYPE& mask, const PACKET_TYPE& a, \ const PACKET_TYPE& b) { \ - return mask != 0 ? a : b; \ + return __builtin_elementwise_abs(mask) == 0 ? b : a; \ } EIGEN_CLANG_PACKET_ELEMENTWISE(Packet16f) @@ -443,9 +506,9 @@ // --- Math functions (float/double only) --- -#if EIGEN_HAS_BUILTIN(__builtin_elementwise_floor) && EIGEN_HAS_BUILTIN(__builtin_elementwise_ceil) && \ - EIGEN_HAS_BUILTIN(__builtin_elementwise_round) && EIGEN_HAS_BUILTIN(__builtin_elementwise_roundeven) && \ - EIGEN_HAS_BUILTIN(__builtin_elementwise_trunc) && EIGEN_HAS_BUILTIN(__builtin_elementwise_sqrt) +#if __has_builtin(__builtin_elementwise_floor) && __has_builtin(__builtin_elementwise_ceil) && \ + __has_builtin(__builtin_elementwise_round) && __has_builtin(__builtin_elementwise_roundeven) && \ + __has_builtin(__builtin_elementwise_trunc) && __has_builtin(__builtin_elementwise_sqrt) #define EIGEN_CLANG_PACKET_MATH_FLOAT(PACKET_TYPE) \ template <> \ EIGEN_STRONG_INLINE PACKET_TYPE pfloor<PACKET_TYPE>(const PACKET_TYPE& a) { \ @@ -478,7 +541,7 @@ #endif // --- Fused Multiply-Add (MADD) --- -#if defined(__FMA__) && EIGEN_HAS_BUILTIN(__builtin_elementwise_fma) +#if defined(__FMA__) && __has_builtin(__builtin_elementwise_fma) #define EIGEN_CLANG_PACKET_MADD(PACKET_TYPE) \ template <> \ EIGEN_STRONG_INLINE PACKET_TYPE pmadd<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \ @@ -526,11 +589,11 @@ EIGEN_STRONG_INLINE PACKET_TYPE pgather<typename unpacket_traits<PACKET_TYPE>::type, PACKET_TYPE>( \ const unpacket_traits<PACKET_TYPE>::type* from, Index stride) { \ constexpr int size = unpacket_traits<PACKET_TYPE>::size; \ - PACKET_TYPE result; \ + unpacket_traits<PACKET_TYPE>::type arr[size]; \ for (int i = 0; i < size; ++i) { \ - result[i] = from[i * stride]; \ + arr[i] = from[i * stride]; \ } \ - return result; \ + return *reinterpret_cast<PACKET_TYPE*>(arr); \ } EIGEN_CLANG_PACKET_SCATTER_GATHER(Packet16f) @@ -540,7 +603,7 @@ #undef EIGEN_CLANG_PACKET_SCATTER_GATHER // ---- Various operations that depend on __builtin_shufflevector. -#if EIGEN_HAS_BUILTIN(__builtin_shufflevector) +#if __has_builtin(__builtin_shufflevector) namespace detail { template <typename Packet> EIGEN_STRONG_INLINE Packet preverse_impl_8(const Packet& a) {
diff --git a/Eigen/src/Core/arch/clang/Reductions.h b/Eigen/src/Core/arch/clang/Reductions.h index d9cc558..8454a9a 100644 --- a/Eigen/src/Core/arch/clang/Reductions.h +++ b/Eigen/src/Core/arch/clang/Reductions.h
@@ -14,8 +14,7 @@ namespace internal { // --- Reductions --- -#if EIGEN_HAS_BUILTIN(__builtin_reduce_min) && EIGEN_HAS_BUILTIN(__builtin_reduce_max) && \ - EIGEN_HAS_BUILTIN(__builtin_reduce_or) +#if __has_builtin(__builtin_reduce_min) && __has_builtin(__builtin_reduce_max) && __has_builtin(__builtin_reduce_or) #define EIGEN_CLANG_PACKET_REDUX_MINMAX(PACKET_TYPE) \ template <> \ EIGEN_STRONG_INLINE unpacket_traits<PACKET_TYPE>::type predux_min(const PACKET_TYPE& a) { \ @@ -37,7 +36,7 @@ #undef EIGEN_CLANG_PACKET_REDUX_MINMAX #endif -#if EIGEN_HAS_BUILTIN(__builtin_reduce_add) && EIGEN_HAS_BUILTIN(__builtin_reduce_mul) +#if __has_builtin(__builtin_reduce_add) && __has_builtin(__builtin_reduce_mul) #define EIGEN_CLANG_PACKET_REDUX_INT(PACKET_TYPE) \ template <> \ EIGEN_STRONG_INLINE unpacket_traits<PACKET_TYPE>::type predux<PACKET_TYPE>(const PACKET_TYPE& a) { \ @@ -54,7 +53,7 @@ #undef EIGEN_CLANG_PACKET_REDUX_INT #endif -#if EIGEN_HAS_BUILTIN(__builtin_shufflevector) +#if __has_builtin(__builtin_shufflevector) namespace detail { template <typename VectorT> EIGEN_STRONG_INLINE scalar_type_of_vector_t<VectorT> ReduceAdd16(const VectorT& a) {
diff --git a/Eigen/src/Core/arch/clang/TypeCasting.h b/Eigen/src/Core/arch/clang/TypeCasting.h index 164056b..0aa4496 100644 --- a/Eigen/src/Core/arch/clang/TypeCasting.h +++ b/Eigen/src/Core/arch/clang/TypeCasting.h
@@ -37,7 +37,7 @@ //============================================================================== // pcast //============================================================================== -#if EIGEN_HAS_BUILTIN(__builtin_convertvector) +#if __has_builtin(__builtin_convertvector) template <> EIGEN_STRONG_INLINE Packet16i pcast<Packet16f, Packet16i>(const Packet16f& a) { return __builtin_convertvector(a, Packet16i);
diff --git a/Eigen/src/Core/functors/UnaryFunctors.h b/Eigen/src/Core/functors/UnaryFunctors.h index d7fc7bb..202995f 100644 --- a/Eigen/src/Core/functors/UnaryFunctors.h +++ b/Eigen/src/Core/functors/UnaryFunctors.h
@@ -106,6 +106,26 @@ }; }; +template <typename Scalar, bool IsComplex = NumTraits<Scalar>::IsComplex> +struct squared_norm_functor { + typedef Scalar result_type; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator()(const Scalar& a) const { + return Scalar(numext::real(a) * numext::real(a), numext::imag(a) * numext::imag(a)); + } + template <typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet packetOp(const Packet& a) const { + return Packet(pmul(a.v, a.v)); + } +}; +template <typename Scalar> +struct squared_norm_functor<Scalar, false> : scalar_abs2_op<Scalar> {}; + +template <typename Scalar> +struct functor_traits<squared_norm_functor<Scalar>> { + using Real = typename NumTraits<Scalar>::Real; + enum { Cost = NumTraits<Real>::MulCost, PacketAccess = packet_traits<Real>::HasMul }; +}; + /** \internal * \brief Template functor to compute the conjugate of a complex value *
diff --git a/Eigen/src/Core/util/ForwardDeclarations.h b/Eigen/src/Core/util/ForwardDeclarations.h index e2e7155..e0bc57e 100644 --- a/Eigen/src/Core/util/ForwardDeclarations.h +++ b/Eigen/src/Core/util/ForwardDeclarations.h
@@ -517,9 +517,6 @@ template <typename Packet> struct has_packet_segment : std::false_type {}; - -template <typename T> -struct complex_array_access; } // namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/util/Macros.h b/Eigen/src/Core/util/Macros.h index 2cec29b..cb00082 100644 --- a/Eigen/src/Core/util/Macros.h +++ b/Eigen/src/Core/util/Macros.h
@@ -704,13 +704,6 @@ #define EIGEN_HAS_BUILTIN(x) 0 #endif -// Cross compiler wrapper around LLVM's __has_attribute -#ifdef __has_attribute -#define EIGEN_HAS_ATTRIBUTE(x) __has_attribute(x) -#else -#define EIGEN_HAS_ATTRIBUTE(x) 0 -#endif - // A Clang feature extension to determine compiler features. // We use it to determine 'cxx_rvalue_references' #ifndef __has_feature @@ -842,7 +835,7 @@ #endif // Does the compiler support vector types? -#if EIGEN_HAS_ATTRIBUTE(ext_vector_type) && EIGEN_HAS_BUILTIN(__builtin_vectorelements) +#if __has_attribute(ext_vector_type) && __has_builtin(__builtin_vectorelements) #define EIGEN_ARCH_VECTOR_EXTENSIONS 1 #else #define EIGEN_ARCH_VECTOR_EXTENSIONS 0
diff --git a/test/realview.cpp b/test/realview.cpp index 7cefe6c..8658a3f 100644 --- a/test/realview.cpp +++ b/test/realview.cpp
@@ -9,20 +9,8 @@ #include "main.h" -// wrapper that disables array-oriented access to real and imaginary components -struct TestComplex : public std::complex<float> { - TestComplex() = default; - TestComplex(const TestComplex&) = default; - TestComplex(std::complex<float> x) : std::complex<float>(x){}; - TestComplex(float x) : std::complex<float>(x){}; -}; -template <> -struct NumTraits<TestComplex> : NumTraits<std::complex<float>> {}; -template <> -struct internal::random_impl<TestComplex> : internal::random_impl<std::complex<float>> {}; - template <typename T> -void test_realview_readonly(const T&) { +void test_realview(const T&) { using Scalar = typename T::Scalar; using RealScalar = typename NumTraits<Scalar>::Real; @@ -38,80 +26,36 @@ Index rows = internal::random<Index>(minRows, maxRows); Index cols = internal::random<Index>(minCols, maxCols); - T A(rows, cols), B(rows, cols); + T A(rows, cols), B, C; VERIFY(A.realView().rows() == rowFactor * A.rows()); VERIFY(A.realView().cols() == colFactor * A.cols()); VERIFY(A.realView().size() == sizeFactor * A.size()); - A.setRandom(); - VERIFY_IS_APPROX(A.matrix().cwiseAbs2().sum(), A.realView().matrix().cwiseAbs2().sum()); - RealScalar alpha = internal::random(RealScalar(1), RealScalar(2)); + A.setRandom(); - // B = A * alpha - for (Index r = 0; r < rows; r++) { - for (Index c = 0; c < cols; c++) { - B.coeffRef(r, c) = A.coeff(r, c) * Scalar(alpha); - } - } - VERIFY_IS_CWISE_APPROX(B.realView(), A.realView() * alpha); - - // B = A / alpha - for (Index r = 0; r < rows; r++) { - for (Index c = 0; c < cols; c++) { - B.coeffRef(r, c) = A.coeff(r, c) / Scalar(alpha); - } - } - VERIFY_IS_CWISE_APPROX(B.realView(), A.realView() / alpha); -} - -template <typename T> -void test_realview(const T&) { - using Scalar = typename T::Scalar; - using RealScalar = typename NumTraits<Scalar>::Real; - - constexpr Index minRows = T::RowsAtCompileTime == Dynamic ? 1 : T::RowsAtCompileTime; - constexpr Index maxRows = T::MaxRowsAtCompileTime == Dynamic ? (EIGEN_TEST_MAX_SIZE / 2) : T::MaxRowsAtCompileTime; - constexpr Index minCols = T::ColsAtCompileTime == Dynamic ? 1 : T::ColsAtCompileTime; - constexpr Index maxCols = T::MaxColsAtCompileTime == Dynamic ? (EIGEN_TEST_MAX_SIZE / 2) : T::MaxColsAtCompileTime; - - constexpr Index rowFactor = (NumTraits<Scalar>::IsComplex && !T::IsRowMajor) ? 2 : 1; - constexpr Index colFactor = (NumTraits<Scalar>::IsComplex && T::IsRowMajor) ? 2 : 1; - constexpr Index sizeFactor = NumTraits<Scalar>::IsComplex ? 2 : 1; - - const Index rows = internal::random<Index>(minRows, maxRows); - const Index cols = internal::random<Index>(minCols, maxCols); - const Index realViewRows = rowFactor * rows; - const Index realViewCols = colFactor * cols; - - const T A = T::Random(rows, cols); - T B; - - VERIFY_IS_EQUAL(A.realView().rows(), rowFactor * A.rows()); - VERIFY_IS_EQUAL(A.realView().cols(), colFactor * A.cols()); - VERIFY_IS_EQUAL(A.realView().size(), sizeFactor * A.size()); - - VERIFY_IS_APPROX(A.matrix().cwiseAbs2().sum(), A.realView().matrix().cwiseAbs2().sum()); + VERIFY_IS_APPROX(A.matrix().squaredNorm(), A.realView().matrix().squaredNorm()); // test re-sizing realView during assignment B.realView() = A.realView(); VERIFY_IS_APPROX(A, B); VERIFY_IS_APPROX(A.realView(), B.realView()); - const RealScalar alpha = internal::random(RealScalar(1), RealScalar(2)); - // B = A * alpha for (Index r = 0; r < rows; r++) { for (Index c = 0; c < cols; c++) { B.coeffRef(r, c) = A.coeff(r, c) * Scalar(alpha); } } - VERIFY_IS_APPROX(B.realView(), A.realView() * alpha); - B = A; - B.realView() *= alpha; VERIFY_IS_APPROX(B.realView(), A.realView() * alpha); + C = A; + C.realView() *= alpha; + VERIFY_IS_APPROX(B, C); + + alpha = internal::random(RealScalar(1), RealScalar(2)); + A.setRandom(); // B = A / alpha for (Index r = 0; r < rows; r++) { @@ -119,99 +63,15 @@ B.coeffRef(r, c) = A.coeff(r, c) / Scalar(alpha); } } + VERIFY_IS_APPROX(B.realView(), A.realView() / alpha); - - B = A; - B.realView() /= alpha; - VERIFY_IS_APPROX(B.realView(), A.realView() / alpha); - - // force some usual access patterns - Index malloc_size = (rows * cols * sizeof(Scalar)) + sizeof(RealScalar); - void* data1 = internal::aligned_malloc(malloc_size); - void* data2 = internal::aligned_malloc(malloc_size); - Scalar* ptr1 = reinterpret_cast<Scalar*>(reinterpret_cast<uint8_t*>(data1) + sizeof(RealScalar)); - Scalar* ptr2 = reinterpret_cast<Scalar*>(reinterpret_cast<uint8_t*>(data2) + sizeof(RealScalar)); - Map<T> C(ptr1, rows, cols), D(ptr2, rows, cols); - - C.setRandom(); - D.setRandom(); - for (Index r = 0; r < realViewRows; r++) { - for (Index c = 0; c < realViewCols; c++) { - C.realView().coeffRef(r, c) = D.realView().coeff(r, c); - } - } - VERIFY_IS_CWISE_EQUAL(C, D); - C = A; - - for (Index c = 0; c < realViewCols - 1; c++) { - B.realView().row(0).coeffRef(realViewCols - 1 - c) = C.realView().row(0).coeff(c + 1); - } - D.realView().row(0).tail(realViewCols - 1) = C.realView().row(0).tail(realViewCols - 1).reverse(); - VERIFY_IS_CWISE_EQUAL(B.realView().row(0).tail(realViewCols - 1), D.realView().row(0).tail(realViewCols - 1)); - - for (Index r = 0; r < realViewRows - 1; r++) { - B.realView().col(0).coeffRef(realViewRows - 1 - r) = C.realView().col(0).coeff(r + 1); - } - D.realView().col(0).tail(realViewRows - 1) = C.realView().col(0).tail(realViewRows - 1).reverse(); - VERIFY_IS_CWISE_EQUAL(B.realView().col(0).tail(realViewRows - 1), D.realView().col(0).tail(realViewRows - 1)); -} - -template <typename ComplexScalar, bool Enable = internal::packet_traits<ComplexScalar>::Vectorizable> -struct test_edge_cases_impl { - static void run() { - using namespace internal; - using RealScalar = typename NumTraits<ComplexScalar>::Real; - using ComplexPacket = typename packet_traits<ComplexScalar>::type; - using RealPacket = typename unpacket_traits<ComplexPacket>::as_real; - constexpr int ComplexSize = unpacket_traits<ComplexPacket>::size; - constexpr int RealSize = 2 * ComplexSize; - VectorX<ComplexScalar> a_data(2 * ComplexSize); - Map<const VectorX<RealScalar>> a_data_asreal(reinterpret_cast<const RealScalar*>(a_data.data()), 2 * a_data.size()); - VectorX<RealScalar> b_data(RealSize); - - a_data.setRandom(); - evaluator<RealView<VectorX<ComplexScalar>>> eval(a_data.realView()); - - for (Index offset = 0; offset < RealSize; offset++) { - for (Index begin = 0; offset + begin < RealSize; begin++) { - for (Index count = 0; begin + count < RealSize; count++) { - b_data.setRandom(); - RealPacket res = eval.template packetSegment<Unaligned, RealPacket>(offset, begin, count); - pstoreSegment(b_data.data(), res, begin, count); - VERIFY_IS_CWISE_EQUAL(a_data_asreal.segment(offset + begin, count), b_data.segment(begin, count)); - } - } - } - } -}; - -template <typename ComplexScalar> -struct test_edge_cases_impl<ComplexScalar, false> { - static void run() {} -}; - -template <typename ComplexScalar> -void test_edge_cases(const ComplexScalar&) { - test_edge_cases_impl<ComplexScalar>::run(); + C.realView() /= alpha; + VERIFY_IS_APPROX(B, C); } template <typename Scalar, int Rows, int Cols, int MaxRows = Rows, int MaxCols = Cols> -void test_realview_readonly() { - // if Rows == 1, don't test ColMajor as it is not a valid array - using ColMajorMatrixType = Matrix<Scalar, Rows, Cols, Rows == 1 ? RowMajor : ColMajor, MaxRows, MaxCols>; - using ColMajorArrayType = Array<Scalar, Rows, Cols, Rows == 1 ? RowMajor : ColMajor, MaxRows, MaxCols>; - // if Cols == 1, don't test RowMajor as it is not a valid array - using RowMajorMatrixType = Matrix<Scalar, Rows, Cols, Cols == 1 ? ColMajor : RowMajor, MaxRows, MaxCols>; - using RowMajorArrayType = Array<Scalar, Rows, Cols, Cols == 1 ? ColMajor : RowMajor, MaxRows, MaxCols>; - test_realview_readonly(ColMajorMatrixType()); - test_realview_readonly(ColMajorArrayType()); - test_realview_readonly(RowMajorMatrixType()); - test_realview_readonly(RowMajorArrayType()); -} - -template <typename Scalar, int Rows, int Cols, int MaxRows = Rows, int MaxCols = Cols> -void test_realview_readwrite() { +void test_realview_driver() { // if Rows == 1, don't test ColMajor as it is not a valid array using ColMajorMatrixType = Matrix<Scalar, Rows, Cols, Rows == 1 ? RowMajor : ColMajor, MaxRows, MaxCols>; using ColMajorArrayType = Array<Scalar, Rows, Cols, Rows == 1 ? RowMajor : ColMajor, MaxRows, MaxCols>; @@ -225,29 +85,26 @@ } template <int Rows, int Cols, int MaxRows = Rows, int MaxCols = Cols> -void test_realview() { - test_realview_readwrite<float, Rows, Cols, MaxRows, MaxCols>(); - test_realview_readwrite<std::complex<float>, Rows, Cols, MaxRows, MaxCols>(); - test_realview_readwrite<double, Rows, Cols, MaxRows, MaxCols>(); - test_realview_readwrite<std::complex<double>, Rows, Cols, MaxRows, MaxCols>(); - test_realview_readwrite<long double, Rows, Cols, MaxRows, MaxCols>(); - test_realview_readwrite<std::complex<long double>, Rows, Cols, MaxRows, MaxCols>(); - test_realview_readonly<TestComplex, Rows, Cols, MaxRows, MaxCols>(); +void test_realview_driver_complex() { + test_realview_driver<float, Rows, Cols, MaxRows, MaxCols>(); + test_realview_driver<std::complex<float>, Rows, Cols, MaxRows, MaxCols>(); + test_realview_driver<double, Rows, Cols, MaxRows, MaxCols>(); + test_realview_driver<std::complex<double>, Rows, Cols, MaxRows, MaxCols>(); + test_realview_driver<long double, Rows, Cols, MaxRows, MaxCols>(); + test_realview_driver<std::complex<long double>, Rows, Cols, MaxRows, MaxCols>(); } EIGEN_DECLARE_TEST(realview) { for (int i = 0; i < g_repeat; i++) { - CALL_SUBTEST_1((test_realview<Dynamic, Dynamic, Dynamic, Dynamic>())); - CALL_SUBTEST_2((test_realview<Dynamic, Dynamic, 17, Dynamic>())); - CALL_SUBTEST_3((test_realview<Dynamic, Dynamic, Dynamic, 19>())); - CALL_SUBTEST_4((test_realview<Dynamic, Dynamic, 17, 19>())); - CALL_SUBTEST_5((test_realview<17, Dynamic, 17, Dynamic>())); - CALL_SUBTEST_6((test_realview<Dynamic, 19, Dynamic, 19>())); - CALL_SUBTEST_7((test_realview<17, 19, 17, 19>())); - CALL_SUBTEST_8((test_realview<Dynamic, 1>())); - CALL_SUBTEST_9((test_realview<1, Dynamic>())); - CALL_SUBTEST_10((test_realview<1, 1>())); - CALL_SUBTEST_11(test_edge_cases(std::complex<float>())); - CALL_SUBTEST_12(test_edge_cases(std::complex<double>())); + CALL_SUBTEST_1((test_realview_driver_complex<Dynamic, Dynamic, Dynamic, Dynamic>())); + CALL_SUBTEST_2((test_realview_driver_complex<Dynamic, Dynamic, 17, Dynamic>())); + CALL_SUBTEST_3((test_realview_driver_complex<Dynamic, Dynamic, Dynamic, 19>())); + CALL_SUBTEST_4((test_realview_driver_complex<Dynamic, Dynamic, 17, 19>())); + CALL_SUBTEST_5((test_realview_driver_complex<17, Dynamic, 17, Dynamic>())); + CALL_SUBTEST_6((test_realview_driver_complex<Dynamic, 19, Dynamic, 19>())); + CALL_SUBTEST_7((test_realview_driver_complex<17, 19, 17, 19>())); + CALL_SUBTEST_8((test_realview_driver_complex<Dynamic, 1>())); + CALL_SUBTEST_9((test_realview_driver_complex<1, Dynamic>())); + CALL_SUBTEST_10((test_realview_driver_complex<1, 1>())); } }