Update Eigen to commit:4be7e6b4e0a82853e853c0c7c4ef72f395e1f497 CHANGELOG ========= 4be7e6b4e - Fix pcmp_* for HVX to comply with the new definition of true = Scalar(1) edcf4c135 - Remove fortran dependency for eigenblas. e4493233e - Fix EIGEN_OPTIMIZATION_BARRIER for clang-cl f5ead2d34 - Fix intel packet math header inclusion order 1e65707aa - Suppress Warray-bounds warning in generic ploaduSegment, fix edge case for vectorized cast abeba8535 - Use proper float literals in SpecialFunctionsImpl.h. b5bef9dcb - Fix bug in Erfc introduced in !1862. 97c7cc620 - Explicitly use the packet trait HasPow to control whether Pow is vectorized. efe5b6979 - Unconditionally include <memory>. Some c++20 builds are currently broken because it is needed for std::assume_aligned. 2cf66d4b0 - Use numext::fma in more places in SparseCore. d7fa5ebe0 - Fix API incompatibility for ILU in superLU support cedf1f4c1 - Fix typo: duplicated '\''for'\'' in docs 302fc46bc - arm packet alignment requirements and aligned loads/stores 430e35fbd - Fixed -Wshadow warning by renaming variables bd0cd1d67 - Fix self-adjoint products when multiplying by a compile-time vector. 6854da2ea - Fix 1x1 selfadjoint matrix-vector product bug ac1b29f82 - Set CMake POLICY CMP0177 to NEW 849a33624 - Move default builds/tests to GitLab runners. 8ac2fb077 - Use numext::fma for sparse x dense dot product. cc0be0043 - Fix docs build. f169c13d8 - Replace PPC g++-10 with g++14. 7fa069ef9 - tensor documentation 7c636dd5d - Move HIP/CUDA defines to Core. 26616fe5b - Fix VSX packetmath psin and pcast tests. a395ee162 - Fix a collection of random failures encountered when testing with Bazel. 0bce653ef - Use QEMU for arm and ppc tests. db8bd5b82 - Modify pselect and various masks to use Scalar(1) for true. 6de0515fa - Create a changelog file. 98fbf6ed7 - Decommission aarch64 ampere runner. PiperOrigin-RevId: 791772929 Change-Id: Iccf81d31e1ae3e4e57b7fee8c55561ac83fc03d0
diff --git a/Eigen/Core b/Eigen/Core index cf2b164..cc003b0 100644 --- a/Eigen/Core +++ b/Eigen/Core
@@ -92,6 +92,7 @@ #include <algorithm> #include <array> +#include <memory> #include <vector> // for std::is_nothrow_move_assignable @@ -121,7 +122,6 @@ #undef isfinite #include <CL/sycl.hpp> #include <map> -#include <memory> #include <thread> #include <utility> #ifndef EIGEN_SYCL_LOCAL_THREAD_DIM0 @@ -192,36 +192,51 @@ #include "src/Core/arch/Default/BFloat16.h" #include "src/Core/arch/Default/GenericPacketMathFunctionsFwd.h" -#if defined EIGEN_VECTORIZE_SSE +#if defined EIGEN_VECTORIZE_AVX512 #include "src/Core/arch/SSE/PacketMath.h" #include "src/Core/arch/SSE/Reductions.h" -#include "src/Core/arch/SSE/Complex.h" -#include "src/Core/arch/SSE/TypeCasting.h" -#include "src/Core/arch/SSE/MathFunctions.h" -#endif - -#if defined EIGEN_VECTORIZE_AVX #include "src/Core/arch/AVX/PacketMath.h" #include "src/Core/arch/AVX/Reductions.h" -#include "src/Core/arch/AVX/Complex.h" -#include "src/Core/arch/AVX/TypeCasting.h" -#include "src/Core/arch/AVX/MathFunctions.h" -#endif - -#if defined EIGEN_VECTORIZE_AVX512 #include "src/Core/arch/AVX512/PacketMath.h" #include "src/Core/arch/AVX512/Reductions.h" -#include "src/Core/arch/AVX512/Complex.h" -#include "src/Core/arch/AVX512/TypeCasting.h" -#include "src/Core/arch/AVX512/MathFunctions.h" -#include "src/Core/arch/AVX512/TrsmKernel.h" -#endif - #if defined EIGEN_VECTORIZE_AVX512FP16 #include "src/Core/arch/AVX512/PacketMathFP16.h" +#endif +#include "src/Core/arch/SSE/TypeCasting.h" +#include "src/Core/arch/AVX/TypeCasting.h" +#include "src/Core/arch/AVX512/TypeCasting.h" +#if defined EIGEN_VECTORIZE_AVX512FP16 #include "src/Core/arch/AVX512/TypeCastingFP16.h" +#endif +#include "src/Core/arch/SSE/Complex.h" +#include "src/Core/arch/AVX/Complex.h" +#include "src/Core/arch/AVX512/Complex.h" +#include "src/Core/arch/SSE/MathFunctions.h" +#include "src/Core/arch/AVX/MathFunctions.h" +#include "src/Core/arch/AVX512/MathFunctions.h" +#if defined EIGEN_VECTORIZE_AVX512FP16 #include "src/Core/arch/AVX512/MathFunctionsFP16.h" #endif +#include "src/Core/arch/AVX512/TrsmKernel.h" +#elif defined EIGEN_VECTORIZE_AVX +// Use AVX for floats and doubles, SSE for integers +#include "src/Core/arch/SSE/PacketMath.h" +#include "src/Core/arch/SSE/Reductions.h" +#include "src/Core/arch/SSE/TypeCasting.h" +#include "src/Core/arch/SSE/Complex.h" +#include "src/Core/arch/AVX/PacketMath.h" +#include "src/Core/arch/AVX/Reductions.h" +#include "src/Core/arch/AVX/TypeCasting.h" +#include "src/Core/arch/AVX/Complex.h" +#include "src/Core/arch/SSE/MathFunctions.h" +#include "src/Core/arch/AVX/MathFunctions.h" +#elif defined EIGEN_VECTORIZE_SSE +#include "src/Core/arch/SSE/PacketMath.h" +#include "src/Core/arch/SSE/Reductions.h" +#include "src/Core/arch/SSE/TypeCasting.h" +#include "src/Core/arch/SSE/MathFunctions.h" +#include "src/Core/arch/SSE/Complex.h" +#endif #if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) #include "src/Core/arch/AltiVec/PacketMath.h"
diff --git a/Eigen/src/Core/CoreEvaluators.h b/Eigen/src/Core/CoreEvaluators.h index 63f1895..ec731ac 100644 --- a/Eigen/src/Core/CoreEvaluators.h +++ b/Eigen/src/Core/CoreEvaluators.h
@@ -707,7 +707,7 @@ Index packetOffset = offset * PacketSize; Index actualRow = IsRowMajor ? row : row + packetOffset; Index actualCol = IsRowMajor ? col + packetOffset : col; - eigen_assert(check_array_bounds(actualRow, actualCol, 0, count) && "Array index out of bounds"); + eigen_assert(check_array_bounds(actualRow, actualCol, begin, count) && "Array index out of bounds"); return m_argImpl.template packetSegment<LoadMode, PacketType>(actualRow, actualCol, begin, count); } template <int LoadMode, typename PacketType = SrcPacketType> @@ -715,8 +715,8 @@ Index offset) const { constexpr int PacketSize = unpacket_traits<PacketType>::size; Index packetOffset = offset * PacketSize; - Index actualIndex = index + packetOffset + begin; - eigen_assert(check_array_bounds(actualIndex, 0, count) && "Array index out of bounds"); + Index actualIndex = index + packetOffset; + eigen_assert(check_array_bounds(actualIndex, begin, count) && "Array index out of bounds"); return m_argImpl.template packetSegment<LoadMode, PacketType>(actualIndex, begin, count); }
diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index ab9c0e1..de599a1 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h
@@ -375,7 +375,7 @@ return a && b; } -// In the generic case, memset to all one bits. +// In the generic packet case, memset to all one bits. template <typename Packet, typename EnableIf = void> struct ptrue_impl { static EIGEN_DEVICE_FUNC inline Packet run(const Packet& /*a*/) { @@ -385,19 +385,16 @@ } }; +// Use a value of one for scalars. +template <typename Scalar> +struct ptrue_impl<Scalar, std::enable_if_t<is_scalar<Scalar>::value>> { + static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar&) { return Scalar(1); } +}; + // For booleans, we can only directly set a valid `bool` value to avoid UB. template <> struct ptrue_impl<bool, void> { - static EIGEN_DEVICE_FUNC inline bool run(const bool& /*a*/) { return true; } -}; - -// For non-trivial scalars, set to Scalar(1) (i.e. a non-zero value). -// Although this is technically not a valid bitmask, the scalar path for pselect -// uses a comparison to zero, so this should still work in most cases. We don't -// have another option, since the scalar type requires initialization. -template <typename T> -struct ptrue_impl<T, std::enable_if_t<is_scalar<T>::value && NumTraits<T>::RequireInitialization>> { - static EIGEN_DEVICE_FUNC inline T run(const T& /*a*/) { return T(1); } + static EIGEN_DEVICE_FUNC inline bool run(const bool&) { return true; } }; /** \internal \returns one bits. */ @@ -406,7 +403,7 @@ return ptrue_impl<Packet>::run(a); } -// In the general case, memset to zero. +// In the general packet case, memset to zero. template <typename Packet, typename EnableIf = void> struct pzero_impl { static EIGEN_DEVICE_FUNC inline Packet run(const Packet& /*a*/) { @@ -641,7 +638,7 @@ } }; -#define EIGEN_BINARY_OP_NAN_PROPAGATION(Type, Func) [](const Type& a, const Type& b) { return Func(a, b); } +#define EIGEN_BINARY_OP_NAN_PROPAGATION(Type, Func) [](const Type& aa, const Type& bb) { return Func(aa, bb); } /** \internal \returns the min of \a a and \a b (coeff-wise). If \a a or \b b is NaN, the return value is implementation defined. */ @@ -875,17 +872,29 @@ return a; } +template <typename Packet, typename EnableIf = void> +struct peven_mask_impl { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet&) { + typedef typename unpacket_traits<Packet>::type Scalar; + const size_t n = unpacket_traits<Packet>::size; + EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar elements[n]; + for (size_t i = 0; i < n; ++i) { + memset(elements + i, ((i & 1) == 0 ? 0xff : 0), sizeof(Scalar)); + } + return ploadu<Packet>(elements); + } +}; + +template <typename Scalar> +struct peven_mask_impl<Scalar, std::enable_if_t<is_scalar<Scalar>::value>> { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(const Scalar&) { return Scalar(1); } +}; + /** \internal \returns a packet with constant coefficients \a a, e.g.: (x, 0, x, 0), where x is the value of all 1-bits. */ template <typename Packet> -EIGEN_DEVICE_FUNC inline Packet peven_mask(const Packet& /*a*/) { - typedef typename unpacket_traits<Packet>::type Scalar; - const size_t n = unpacket_traits<Packet>::size; - EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar elements[n]; - for (size_t i = 0; i < n; ++i) { - memset(elements + i, ((i & 1) == 0 ? 0xff : 0), sizeof(Scalar)); - } - return ploadu<Packet>(elements); +EIGEN_DEVICE_FUNC inline Packet peven_mask(const Packet& a) { + return peven_mask_impl<Packet>::run(a); } /** \internal copy the packet \a from to \a *to, \a to must be properly aligned */ @@ -1587,9 +1596,10 @@ using Scalar = typename unpacket_traits<Packet>::type; constexpr Index PacketSize = unpacket_traits<Packet>::size; eigen_assert((begin >= 0 && count >= 0 && begin + count <= PacketSize) && "invalid range"); - Scalar aux[PacketSize]; - memset(static_cast<void*>(aux), 0x00, sizeof(Scalar) * PacketSize); - smart_copy(from + begin, from + begin + count, aux + begin); + Scalar aux[PacketSize] = {}; + for (Index k = begin; k < begin + count; k++) { + aux[k] = from[k]; + } return ploadu<Packet>(aux); } @@ -1610,7 +1620,9 @@ eigen_assert((begin >= 0 && count >= 0 && begin + count <= PacketSize) && "invalid range"); Scalar aux[PacketSize]; pstoreu<Scalar, Packet>(aux, from); - smart_copy(aux + begin, aux + begin + count, to + begin); + for (Index k = begin; k < begin + count; k++) { + to[k] = aux[k]; + } } /** \internal copy the packet \a from in the range [begin, begin + count) to \a *to.
diff --git a/Eigen/src/Core/MathFunctions.h b/Eigen/src/Core/MathFunctions.h index 941961d..481e057 100644 --- a/Eigen/src/Core/MathFunctions.h +++ b/Eigen/src/Core/MathFunctions.h
@@ -182,10 +182,6 @@ typedef typename NumTraits<Scalar>::Real& type; }; -// implementation in MathFunctionsImpl.h -template <typename Mask, bool is_built_in_float = std::is_floating_point<Mask>::value> -struct scalar_select_mask; - } // namespace internal namespace numext { @@ -211,9 +207,9 @@ return EIGEN_MATHFUNC_IMPL(imag, Scalar)::run(x); } -template <typename Scalar, typename Mask> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar select(const Mask& mask, const Scalar& a, const Scalar& b) { - return internal::scalar_select_mask<Mask>::run(mask) ? b : a; +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar select(const Scalar& mask, const Scalar& a, const Scalar& b) { + return numext::is_exactly_zero(mask) ? b : a; } } // namespace numext
diff --git a/Eigen/src/Core/MathFunctionsImpl.h b/Eigen/src/Core/MathFunctionsImpl.h index cbac1c2..c4b5da3 100644 --- a/Eigen/src/Core/MathFunctionsImpl.h +++ b/Eigen/src/Core/MathFunctionsImpl.h
@@ -28,7 +28,7 @@ 2. If a is zero, approx_a_recip must be infinite with the same sign as a. 3. If a is infinite, approx_a_recip must be zero with the same sign as a. - If the preconditions are satisfied, which they are for for the _*_rcp_ps + If the preconditions are satisfied, which they are for the _*_rcp_ps instructions on x86, the result has a maximum relative error of 2 ulps, and correctly handles reciprocals of zero, infinity, and NaN. */ @@ -66,7 +66,7 @@ 2. If a is zero, approx_a_recip must be infinite with the same sign as a. 3. If a is infinite, approx_a_recip must be zero with the same sign as a. - If the preconditions are satisfied, which they are for for the _*_rcp_ps + If the preconditions are satisfied, which they are for the _*_rcp_ps instructions on x86, the result has a maximum relative error of 2 ulps, and correctly handles zero, infinity, and NaN. Positive denormals are treated as zero. @@ -116,7 +116,7 @@ 2. If a is zero, approx_rsqrt must be infinite. 3. If a is infinite, approx_rsqrt must be zero. - If the preconditions are satisfied, which they are for for the _*_rsqrt_ps + If the preconditions are satisfied, which they are for the _*_rsqrt_ps instructions on x86, the result has a maximum relative error of 2 ulps, and correctly handles zero and infinity, and NaN. Positive denormal inputs are treated as zero. @@ -256,48 +256,6 @@ return ComplexT(numext::log(a), b); } -// For generic scalars, use ternary select. -template <typename Mask> -struct scalar_select_mask<Mask, /*is_built_in_float*/ false> { - static EIGEN_DEVICE_FUNC inline bool run(const Mask& mask) { return numext::is_exactly_zero(mask); } -}; - -// For built-in float mask, bitcast the mask to its integer counterpart and use ternary select. -template <typename Mask> -struct scalar_select_mask<Mask, /*is_built_in_float*/ true> { - using IntegerType = typename numext::get_integer_by_size<sizeof(Mask)>::unsigned_type; - static EIGEN_DEVICE_FUNC inline bool run(const Mask& mask) { - return numext::is_exactly_zero(numext::bit_cast<IntegerType>(std::abs(mask))); - } -}; - -template <int Size = sizeof(long double)> -struct ldbl_select_mask { - static constexpr int MantissaDigits = std::numeric_limits<long double>::digits; - static constexpr int NumBytes = (MantissaDigits == 64 ? 80 : 128) / CHAR_BIT; - static EIGEN_DEVICE_FUNC inline bool run(const long double& mask) { - const uint8_t* mask_bytes = reinterpret_cast<const uint8_t*>(&mask); - for (Index i = 0; i < NumBytes; i++) { - if (mask_bytes[i] != 0) return false; - } - return true; - } -}; - -template <> -struct ldbl_select_mask<sizeof(double)> : scalar_select_mask<double> {}; - -template <> -struct scalar_select_mask<long double, true> : ldbl_select_mask<> {}; - -template <typename RealMask> -struct scalar_select_mask<std::complex<RealMask>, false> { - using impl = scalar_select_mask<RealMask>; - static EIGEN_DEVICE_FUNC inline bool run(const std::complex<RealMask>& mask) { - return impl::run(numext::real(mask)) && impl::run(numext::imag(mask)); - } -}; - } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/ProductEvaluators.h b/Eigen/src/Core/ProductEvaluators.h index ce8d954..a230044 100644 --- a/Eigen/src/Core/ProductEvaluators.h +++ b/Eigen/src/Core/ProductEvaluators.h
@@ -846,7 +846,7 @@ template <typename Dest> static EIGEN_DEVICE_FUNC void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) { - selfadjoint_product_impl<typename Lhs::MatrixType, Lhs::Mode, false, Rhs, 0, Rhs::IsVectorAtCompileTime>::run( + selfadjoint_product_impl<typename Lhs::MatrixType, Lhs::Mode, false, Rhs, 0, Rhs::ColsAtCompileTime == 1>::run( dst, lhs.nestedExpression(), rhs, alpha); } }; @@ -858,7 +858,7 @@ template <typename Dest> static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) { - selfadjoint_product_impl<Lhs, 0, Lhs::IsVectorAtCompileTime, typename Rhs::MatrixType, Rhs::Mode, false>::run( + selfadjoint_product_impl<Lhs, 0, Lhs::RowsAtCompileTime == 1, typename Rhs::MatrixType, Rhs::Mode, false>::run( dst, lhs, rhs.nestedExpression(), alpha); } };
diff --git a/Eigen/src/Core/arch/AVX/PacketMath.h b/Eigen/src/Core/arch/AVX/PacketMath.h index eb5da53..0cd9e6c 100644 --- a/Eigen/src/Core/arch/AVX/PacketMath.h +++ b/Eigen/src/Core/arch/AVX/PacketMath.h
@@ -118,6 +118,7 @@ HasLog1p = 1, HasExpm1 = 1, HasExp = 1, + HasPow = 1, HasNdtri = 1, HasBessel = 1, HasSqrt = 1, @@ -149,6 +150,7 @@ HasErf = 1, HasErfc = 1, HasExp = 1, + HasPow = 1, HasSqrt = 1, HasRsqrt = 1, HasCbrt = 1,
diff --git a/Eigen/src/Core/arch/AVX512/PacketMath.h b/Eigen/src/Core/arch/AVX512/PacketMath.h index 932b056..b76c8a7 100644 --- a/Eigen/src/Core/arch/AVX512/PacketMath.h +++ b/Eigen/src/Core/arch/AVX512/PacketMath.h
@@ -135,6 +135,7 @@ HasNdtri = 1, HasBessel = 1, HasExp = 1, + HasPow = 1, HasReciprocal = EIGEN_FAST_MATH, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH, @@ -159,6 +160,7 @@ HasCos = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, + HasPow = 1, HasATan = 1, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH,
diff --git a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h index ef64bc5..a040bbe 100644 --- a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h +++ b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h
@@ -1,1413 +1,1413 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2025 The Eigen Authors. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -#ifndef EIGEN_PACKET_MATH_FP16_AVX512_H -#define EIGEN_PACKET_MATH_FP16_AVX512_H - -// IWYU pragma: private -#include "../../InternalHeaderCheck.h" - -namespace Eigen { - -namespace internal { - -typedef __m512h Packet32h; -typedef __m256h Packet16h; -typedef __m128h Packet8h; - -template <> -struct is_arithmetic<Packet8h> { - enum { value = true }; -}; - -template <> -struct packet_traits<half> : default_packet_traits { - typedef Packet32h type; - typedef Packet16h half; - enum { - Vectorizable = 1, - AlignedOnScalar = 1, - size = 32, - - HasCmp = 1, - HasAdd = 1, - HasSub = 1, - HasMul = 1, - HasDiv = 1, - HasNegate = 1, - HasAbs = 1, - HasAbs2 = 0, - HasMin = 1, - HasMax = 1, - HasConj = 1, - HasSetLinear = 0, - HasLog = 1, - HasLog1p = 1, - HasExp = 1, - HasExpm1 = 1, - HasSqrt = 1, - HasRsqrt = 1, - // These ones should be implemented in future - HasBessel = 0, - HasNdtri = 0, - HasSin = EIGEN_FAST_MATH, - HasCos = EIGEN_FAST_MATH, - HasTanh = EIGEN_FAST_MATH, - HasErf = 0, // EIGEN_FAST_MATH, - HasBlend = 0 - }; -}; - -template <> -struct unpacket_traits<Packet32h> { - typedef Eigen::half type; - typedef Packet16h half; - typedef Packet32s integer_packet; - enum { - size = 32, - alignment = Aligned64, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; - -template <> -struct unpacket_traits<Packet16h> { - typedef Eigen::half type; - typedef Packet8h half; - typedef Packet16s integer_packet; - enum { - size = 16, - alignment = Aligned32, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; - -template <> -struct unpacket_traits<Packet8h> { - typedef Eigen::half type; - typedef Packet8h half; - typedef Packet8s integer_packet; - enum { - size = 8, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; - -// Conversions - -EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) { return _mm512_cvtxph_ps(a); } - -EIGEN_STRONG_INLINE Packet8f half2float(const Packet8h& a) { return _mm256_cvtxph_ps(a); } - -EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) { return _mm512_cvtxps_ph(a); } - -EIGEN_STRONG_INLINE Packet8h float2half(const Packet8f& a) { return _mm256_cvtxps_ph(a); } - -// Memory functions - -// pset1 - -template <> -EIGEN_STRONG_INLINE Packet32h pset1<Packet32h>(const Eigen::half& from) { - return _mm512_set1_ph(from.x); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(const Eigen::half& from) { - return _mm256_set1_ph(from.x); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pset1<Packet8h>(const Eigen::half& from) { - return _mm_set1_ph(from.x); -} - -template <> -EIGEN_STRONG_INLINE Packet32h pzero(const Packet32h& /*a*/) { - return _mm512_setzero_ph(); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pzero(const Packet16h& /*a*/) { - return _mm256_setzero_ph(); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pzero(const Packet8h& /*a*/) { - return _mm_setzero_ph(); -} - -// pset1frombits -template <> -EIGEN_STRONG_INLINE Packet32h pset1frombits<Packet32h>(unsigned short from) { - return _mm512_castsi512_ph(_mm512_set1_epi16(from)); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pset1frombits<Packet16h>(unsigned short from) { - return _mm256_castsi256_ph(_mm256_set1_epi16(from)); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pset1frombits<Packet8h>(unsigned short from) { - return _mm_castsi128_ph(_mm_set1_epi16(from)); -} - -// pfirst - -template <> -EIGEN_STRONG_INLINE Eigen::half pfirst<Packet32h>(const Packet32h& from) { - return Eigen::half(_mm512_cvtsh_h(from)); -} - -template <> -EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(const Packet16h& from) { - return Eigen::half(_mm256_cvtsh_h(from)); -} - -template <> -EIGEN_STRONG_INLINE Eigen::half pfirst<Packet8h>(const Packet8h& from) { - return Eigen::half(_mm_cvtsh_h(from)); -} - -// pload - -template <> -EIGEN_STRONG_INLINE Packet32h pload<Packet32h>(const Eigen::half* from) { - EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ph(from); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(const Eigen::half* from) { - EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_ph(from); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pload<Packet8h>(const Eigen::half* from) { - EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ph(from); -} - -// ploadu - -template <> -EIGEN_STRONG_INLINE Packet32h ploadu<Packet32h>(const Eigen::half* from) { - EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ph(from); -} - -template <> -EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(const Eigen::half* from) { - EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_ph(from); -} - -template <> -EIGEN_STRONG_INLINE Packet8h ploadu<Packet8h>(const Eigen::half* from) { - EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_ph(from); -} - -// pstore - -template <> -EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet32h& from) { - EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ph(to, from); -} - -template <> -EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet16h& from) { - EIGEN_DEBUG_ALIGNED_STORE _mm256_store_ph(to, from); -} - -template <> -EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet8h& from) { - EIGEN_DEBUG_ALIGNED_STORE _mm_store_ph(to, from); -} - -// pstoreu - -template <> -EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet32h& from) { - EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ph(to, from); -} - -template <> -EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet16h& from) { - EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_ph(to, from); -} - -template <> -EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet8h& from) { - EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_ph(to, from); -} - -// ploaddup -template <> -EIGEN_STRONG_INLINE Packet32h ploaddup<Packet32h>(const Eigen::half* from) { - __m512h a = _mm512_castph256_ph512(_mm256_loadu_ph(from)); - return _mm512_permutexvar_ph(_mm512_set_epi16(15, 15, 14, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, 6, 6, - 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0), - a); -} - -template <> -EIGEN_STRONG_INLINE Packet16h ploaddup<Packet16h>(const Eigen::half* from) { - __m256h a = _mm256_castph128_ph256(_mm_loadu_ph(from)); - return _mm256_permutexvar_ph(_mm256_set_epi16(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0), a); -} - -template <> -EIGEN_STRONG_INLINE Packet8h ploaddup<Packet8h>(const Eigen::half* from) { - return _mm_set_ph(from[3].x, from[3].x, from[2].x, from[2].x, from[1].x, from[1].x, from[0].x, from[0].x); -} - -// ploadquad -template <> -EIGEN_STRONG_INLINE Packet32h ploadquad<Packet32h>(const Eigen::half* from) { - __m512h a = _mm512_castph128_ph512(_mm_loadu_ph(from)); - return _mm512_permutexvar_ph( - _mm512_set_epi16(7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0), - a); -} - -template <> -EIGEN_STRONG_INLINE Packet16h ploadquad<Packet16h>(const Eigen::half* from) { - return _mm256_set_ph(from[3].x, from[3].x, from[3].x, from[3].x, from[2].x, from[2].x, from[2].x, from[2].x, - from[1].x, from[1].x, from[1].x, from[1].x, from[0].x, from[0].x, from[0].x, from[0].x); -} - -template <> -EIGEN_STRONG_INLINE Packet8h ploadquad<Packet8h>(const Eigen::half* from) { - return _mm_set_ph(from[1].x, from[1].x, from[1].x, from[1].x, from[0].x, from[0].x, from[0].x, from[0].x); -} - -// pabs - -template <> -EIGEN_STRONG_INLINE Packet32h pabs<Packet32h>(const Packet32h& a) { - return _mm512_abs_ph(a); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pabs<Packet16h>(const Packet16h& a) { - return _mm256_abs_ph(a); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pabs<Packet8h>(const Packet8h& a) { - return _mm_abs_ph(a); -} - -// psignbit - -template <> -EIGEN_STRONG_INLINE Packet32h psignbit<Packet32h>(const Packet32h& a) { - return _mm512_castsi512_ph(_mm512_srai_epi16(_mm512_castph_si512(a), 15)); -} - -template <> -EIGEN_STRONG_INLINE Packet16h psignbit<Packet16h>(const Packet16h& a) { - return _mm256_castsi256_ph(_mm256_srai_epi16(_mm256_castph_si256(a), 15)); -} - -template <> -EIGEN_STRONG_INLINE Packet8h psignbit<Packet8h>(const Packet8h& a) { - return _mm_castsi128_ph(_mm_srai_epi16(_mm_castph_si128(a), 15)); -} - -// pmin - -template <> -EIGEN_STRONG_INLINE Packet32h pmin<Packet32h>(const Packet32h& a, const Packet32h& b) { - return _mm512_min_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pmin<Packet16h>(const Packet16h& a, const Packet16h& b) { - return _mm256_min_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pmin<Packet8h>(const Packet8h& a, const Packet8h& b) { - return _mm_min_ph(a, b); -} - -// pmax - -template <> -EIGEN_STRONG_INLINE Packet32h pmax<Packet32h>(const Packet32h& a, const Packet32h& b) { - return _mm512_max_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pmax<Packet16h>(const Packet16h& a, const Packet16h& b) { - return _mm256_max_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pmax<Packet8h>(const Packet8h& a, const Packet8h& b) { - return _mm_max_ph(a, b); -} - -// plset -template <> -EIGEN_STRONG_INLINE Packet32h plset<Packet32h>(const half& a) { - return _mm512_add_ph(pset1<Packet32h>(a), _mm512_set_ph(31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, - 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); -} - -template <> -EIGEN_STRONG_INLINE Packet16h plset<Packet16h>(const half& a) { - return _mm256_add_ph(pset1<Packet16h>(a), _mm256_set_ph(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); -} - -template <> -EIGEN_STRONG_INLINE Packet8h plset<Packet8h>(const half& a) { - return _mm_add_ph(pset1<Packet8h>(a), _mm_set_ph(7, 6, 5, 4, 3, 2, 1, 0)); -} - -// por - -template <> -EIGEN_STRONG_INLINE Packet32h por(const Packet32h& a, const Packet32h& b) { - return _mm512_castsi512_ph(_mm512_or_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a, const Packet16h& b) { - return _mm256_castsi256_ph(_mm256_or_si256(_mm256_castph_si256(a), _mm256_castph_si256(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet8h por(const Packet8h& a, const Packet8h& b) { - return _mm_castsi128_ph(_mm_or_si128(_mm_castph_si128(a), _mm_castph_si128(b))); -} - -// pxor - -template <> -EIGEN_STRONG_INLINE Packet32h pxor(const Packet32h& a, const Packet32h& b) { - return _mm512_castsi512_ph(_mm512_xor_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a, const Packet16h& b) { - return _mm256_castsi256_ph(_mm256_xor_si256(_mm256_castph_si256(a), _mm256_castph_si256(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pxor(const Packet8h& a, const Packet8h& b) { - return _mm_castsi128_ph(_mm_xor_si128(_mm_castph_si128(a), _mm_castph_si128(b))); -} - -// pand - -template <> -EIGEN_STRONG_INLINE Packet32h pand(const Packet32h& a, const Packet32h& b) { - return _mm512_castsi512_ph(_mm512_and_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a, const Packet16h& b) { - return _mm256_castsi256_ph(_mm256_and_si256(_mm256_castph_si256(a), _mm256_castph_si256(b))); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pand(const Packet8h& a, const Packet8h& b) { - return _mm_castsi128_ph(_mm_and_si128(_mm_castph_si128(a), _mm_castph_si128(b))); -} - -// pandnot - -template <> -EIGEN_STRONG_INLINE Packet32h pandnot(const Packet32h& a, const Packet32h& b) { - return _mm512_castsi512_ph(_mm512_andnot_si512(_mm512_castph_si512(b), _mm512_castph_si512(a))); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a, const Packet16h& b) { - return _mm256_castsi256_ph(_mm256_andnot_si256(_mm256_castph_si256(b), _mm256_castph_si256(a))); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pandnot(const Packet8h& a, const Packet8h& b) { - return _mm_castsi128_ph(_mm_andnot_si128(_mm_castph_si128(b), _mm_castph_si128(a))); -} - -// pselect - -template <> -EIGEN_DEVICE_FUNC inline Packet32h pselect(const Packet32h& mask, const Packet32h& a, const Packet32h& b) { - __mmask32 mask32 = _mm512_cmp_epi16_mask(_mm512_castph_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ); - return _mm512_mask_blend_ph(mask32, a, b); -} - -template <> -EIGEN_DEVICE_FUNC inline Packet16h pselect(const Packet16h& mask, const Packet16h& a, const Packet16h& b) { - __mmask16 mask16 = _mm256_cmp_epi16_mask(_mm256_castph_si256(mask), _mm256_setzero_si256(), _MM_CMPINT_EQ); - return _mm256_mask_blend_ph(mask16, a, b); -} - -template <> -EIGEN_DEVICE_FUNC inline Packet8h pselect(const Packet8h& mask, const Packet8h& a, const Packet8h& b) { - __mmask8 mask8 = _mm_cmp_epi16_mask(_mm_castph_si128(mask), _mm_setzero_si128(), _MM_CMPINT_EQ); - return _mm_mask_blend_ph(mask8, a, b); -} - -// pcmp_eq - -template <> -EIGEN_STRONG_INLINE Packet32h pcmp_eq(const Packet32h& a, const Packet32h& b) { - __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_EQ_OQ); - return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a, const Packet16h& b) { - __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_EQ_OQ); - return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pcmp_eq(const Packet8h& a, const Packet8h& b) { - __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_EQ_OQ); - return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -// pcmp_le - -template <> -EIGEN_STRONG_INLINE Packet32h pcmp_le(const Packet32h& a, const Packet32h& b) { - __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LE_OQ); - return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pcmp_le(const Packet16h& a, const Packet16h& b) { - __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_LE_OQ); - return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pcmp_le(const Packet8h& a, const Packet8h& b) { - __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_LE_OQ); - return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -// pcmp_lt - -template <> -EIGEN_STRONG_INLINE Packet32h pcmp_lt(const Packet32h& a, const Packet32h& b) { - __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LT_OQ); - return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pcmp_lt(const Packet16h& a, const Packet16h& b) { - __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_LT_OQ); - return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pcmp_lt(const Packet8h& a, const Packet8h& b) { - __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_LT_OQ); - return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -// pcmp_lt_or_nan - -template <> -EIGEN_STRONG_INLINE Packet32h pcmp_lt_or_nan(const Packet32h& a, const Packet32h& b) { - __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_NGE_UQ); - return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi16(0), mask, static_cast<short>(0xffffu))); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pcmp_lt_or_nan(const Packet16h& a, const Packet16h& b) { - __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_NGE_UQ); - return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pcmp_lt_or_nan(const Packet8h& a, const Packet8h& b) { - __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_NGE_UQ); - return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast<short>(0xffffu))); -} - -// padd - -template <> -EIGEN_STRONG_INLINE Packet32h padd<Packet32h>(const Packet32h& a, const Packet32h& b) { - return _mm512_add_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) { - return _mm256_add_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet8h padd<Packet8h>(const Packet8h& a, const Packet8h& b) { - return _mm_add_ph(a, b); -} - -// psub - -template <> -EIGEN_STRONG_INLINE Packet32h psub<Packet32h>(const Packet32h& a, const Packet32h& b) { - return _mm512_sub_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet16h psub<Packet16h>(const Packet16h& a, const Packet16h& b) { - return _mm256_sub_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet8h psub<Packet8h>(const Packet8h& a, const Packet8h& b) { - return _mm_sub_ph(a, b); -} - -// pmul - -template <> -EIGEN_STRONG_INLINE Packet32h pmul<Packet32h>(const Packet32h& a, const Packet32h& b) { - return _mm512_mul_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) { - return _mm256_mul_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pmul<Packet8h>(const Packet8h& a, const Packet8h& b) { - return _mm_mul_ph(a, b); -} - -// pdiv - -template <> -EIGEN_STRONG_INLINE Packet32h pdiv<Packet32h>(const Packet32h& a, const Packet32h& b) { - return _mm512_div_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pdiv<Packet16h>(const Packet16h& a, const Packet16h& b) { - return _mm256_div_ph(a, b); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pdiv<Packet8h>(const Packet8h& a, const Packet8h& b) { - return _mm_div_ph(a, b); - ; -} - -// pround - -template <> -EIGEN_STRONG_INLINE Packet32h pround<Packet32h>(const Packet32h& a) { - // Work-around for default std::round rounding mode. - - // Mask for the sign bit. - const Packet32h signMask = - pset1frombits<Packet32h>(static_cast<numext::uint16_t>(static_cast<std::uint16_t>(0x8000u))); - // The largest half-precision float less than 0.5. - const Packet32h prev0dot5 = pset1frombits<Packet32h>(static_cast<numext::uint16_t>(0x37FFu)); - - return _mm512_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pround<Packet16h>(const Packet16h& a) { - // Work-around for default std::round rounding mode. - - // Mask for the sign bit. - const Packet16h signMask = - pset1frombits<Packet16h>(static_cast<numext::uint16_t>(static_cast<std::uint16_t>(0x8000u))); - // The largest half-precision float less than 0.5. - const Packet16h prev0dot5 = pset1frombits<Packet16h>(static_cast<numext::uint16_t>(0x37FFu)); - - return _mm256_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pround<Packet8h>(const Packet8h& a) { - // Work-around for default std::round rounding mode. - - // Mask for the sign bit. - const Packet8h signMask = pset1frombits<Packet8h>(static_cast<numext::uint16_t>(static_cast<std::uint16_t>(0x8000u))); - // The largest half-precision float less than 0.5. - const Packet8h prev0dot5 = pset1frombits<Packet8h>(static_cast<numext::uint16_t>(0x37FFu)); - - return _mm_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO); -} - -// print - -template <> -EIGEN_STRONG_INLINE Packet32h print<Packet32h>(const Packet32h& a) { - return _mm512_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION); -} - -template <> -EIGEN_STRONG_INLINE Packet16h print<Packet16h>(const Packet16h& a) { - return _mm256_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION); -} - -template <> -EIGEN_STRONG_INLINE Packet8h print<Packet8h>(const Packet8h& a) { - return _mm_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION); -} - -// pceil - -template <> -EIGEN_STRONG_INLINE Packet32h pceil<Packet32h>(const Packet32h& a) { - return _mm512_roundscale_ph(a, _MM_FROUND_TO_POS_INF); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pceil<Packet16h>(const Packet16h& a) { - return _mm256_roundscale_ph(a, _MM_FROUND_TO_POS_INF); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pceil<Packet8h>(const Packet8h& a) { - return _mm_roundscale_ph(a, _MM_FROUND_TO_POS_INF); -} - -// pfloor - -template <> -EIGEN_STRONG_INLINE Packet32h pfloor<Packet32h>(const Packet32h& a) { - return _mm512_roundscale_ph(a, _MM_FROUND_TO_NEG_INF); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pfloor<Packet16h>(const Packet16h& a) { - return _mm256_roundscale_ph(a, _MM_FROUND_TO_NEG_INF); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pfloor<Packet8h>(const Packet8h& a) { - return _mm_roundscale_ph(a, _MM_FROUND_TO_NEG_INF); -} - -// ptrunc - -template <> -EIGEN_STRONG_INLINE Packet32h ptrunc<Packet32h>(const Packet32h& a) { - return _mm512_roundscale_ph(a, _MM_FROUND_TO_ZERO); -} - -template <> -EIGEN_STRONG_INLINE Packet16h ptrunc<Packet16h>(const Packet16h& a) { - return _mm256_roundscale_ph(a, _MM_FROUND_TO_ZERO); -} - -template <> -EIGEN_STRONG_INLINE Packet8h ptrunc<Packet8h>(const Packet8h& a) { - return _mm_roundscale_ph(a, _MM_FROUND_TO_ZERO); -} - -// predux -template <> -EIGEN_STRONG_INLINE half predux<Packet32h>(const Packet32h& a) { - return half(_mm512_reduce_add_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE half predux<Packet16h>(const Packet16h& a) { - return half(_mm256_reduce_add_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE half predux<Packet8h>(const Packet8h& a) { - return half(_mm_reduce_add_ph(a)); -} - -// predux_half_dowto4 -template <> -EIGEN_STRONG_INLINE Packet16h predux_half_dowto4<Packet32h>(const Packet32h& a) { - const __m512i bits = _mm512_castph_si512(a); - Packet16h lo = _mm256_castsi256_ph(_mm512_castsi512_si256(bits)); - Packet16h hi = _mm256_castsi256_ph(_mm512_extracti64x4_epi64(bits, 1)); - return padd(lo, hi); -} - -template <> -EIGEN_STRONG_INLINE Packet8h predux_half_dowto4<Packet16h>(const Packet16h& a) { - Packet8h lo = _mm_castsi128_ph(_mm256_castsi256_si128(_mm256_castph_si256(a))); - Packet8h hi = _mm_castps_ph(_mm256_extractf128_ps(_mm256_castph_ps(a), 1)); - return padd(lo, hi); -} - -// predux_max - -template <> -EIGEN_STRONG_INLINE half predux_max<Packet32h>(const Packet32h& a) { - return half(_mm512_reduce_max_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE half predux_max<Packet16h>(const Packet16h& a) { - return half(_mm256_reduce_max_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE half predux_max<Packet8h>(const Packet8h& a) { - return half(_mm_reduce_max_ph(a)); -} - -// predux_min - -template <> -EIGEN_STRONG_INLINE half predux_min<Packet32h>(const Packet32h& a) { - return half(_mm512_reduce_min_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE half predux_min<Packet16h>(const Packet16h& a) { - return half(_mm256_reduce_min_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE half predux_min<Packet8h>(const Packet8h& a) { - return half(_mm_reduce_min_ph(a)); -} - -// predux_mul - -template <> -EIGEN_STRONG_INLINE half predux_mul<Packet32h>(const Packet32h& a) { - return half(_mm512_reduce_mul_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE half predux_mul<Packet16h>(const Packet16h& a) { - return half(_mm256_reduce_mul_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE half predux_mul<Packet8h>(const Packet8h& a) { - return half(_mm_reduce_mul_ph(a)); -} - -#ifdef EIGEN_VECTORIZE_FMA - -// pmadd - -template <> -EIGEN_STRONG_INLINE Packet32h pmadd(const Packet32h& a, const Packet32h& b, const Packet32h& c) { - return _mm512_fmadd_ph(a, b, c); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pmadd(const Packet16h& a, const Packet16h& b, const Packet16h& c) { - return _mm256_fmadd_ph(a, b, c); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pmadd(const Packet8h& a, const Packet8h& b, const Packet8h& c) { - return _mm_fmadd_ph(a, b, c); -} - -// pmsub - -template <> -EIGEN_STRONG_INLINE Packet32h pmsub(const Packet32h& a, const Packet32h& b, const Packet32h& c) { - return _mm512_fmsub_ph(a, b, c); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pmsub(const Packet16h& a, const Packet16h& b, const Packet16h& c) { - return _mm256_fmsub_ph(a, b, c); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pmsub(const Packet8h& a, const Packet8h& b, const Packet8h& c) { - return _mm_fmsub_ph(a, b, c); -} - -// pnmadd - -template <> -EIGEN_STRONG_INLINE Packet32h pnmadd(const Packet32h& a, const Packet32h& b, const Packet32h& c) { - return _mm512_fnmadd_ph(a, b, c); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pnmadd(const Packet16h& a, const Packet16h& b, const Packet16h& c) { - return _mm256_fnmadd_ph(a, b, c); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pnmadd(const Packet8h& a, const Packet8h& b, const Packet8h& c) { - return _mm_fnmadd_ph(a, b, c); -} - -// pnmsub - -template <> -EIGEN_STRONG_INLINE Packet32h pnmsub(const Packet32h& a, const Packet32h& b, const Packet32h& c) { - return _mm512_fnmsub_ph(a, b, c); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pnmsub(const Packet16h& a, const Packet16h& b, const Packet16h& c) { - return _mm256_fnmsub_ph(a, b, c); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pnmsub(const Packet8h& a, const Packet8h& b, const Packet8h& c) { - return _mm_fnmsub_ph(a, b, c); -} - -#endif - -// pnegate - -template <> -EIGEN_STRONG_INLINE Packet32h pnegate<Packet32h>(const Packet32h& a) { - return _mm512_castsi512_ph( - _mm512_xor_si512(_mm512_castph_si512(a), _mm512_set1_epi16(static_cast<std::uint16_t>(0x8000u)))); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pnegate<Packet16h>(const Packet16h& a) { - return _mm256_castsi256_ph( - _mm256_xor_si256(_mm256_castph_si256(a), _mm256_set1_epi16(static_cast<std::uint16_t>(0x8000u)))); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pnegate<Packet8h>(const Packet8h& a) { - return _mm_castsi128_ph(_mm_xor_si128(_mm_castph_si128(a), _mm_set1_epi16(static_cast<std::uint16_t>(0x8000u)))); -} - -// pconj - -// Nothing, packets are real. - -// psqrt - -template <> -EIGEN_STRONG_INLINE Packet32h psqrt<Packet32h>(const Packet32h& a) { - return generic_sqrt_newton_step<Packet32h>::run(a, _mm512_rsqrt_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE Packet16h psqrt<Packet16h>(const Packet16h& a) { - return generic_sqrt_newton_step<Packet16h>::run(a, _mm256_rsqrt_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE Packet8h psqrt<Packet8h>(const Packet8h& a) { - return generic_sqrt_newton_step<Packet8h>::run(a, _mm_rsqrt_ph(a)); -} - -// prsqrt - -template <> -EIGEN_STRONG_INLINE Packet32h prsqrt<Packet32h>(const Packet32h& a) { - return generic_rsqrt_newton_step<Packet32h, /*Steps=*/1>::run(a, _mm512_rsqrt_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE Packet16h prsqrt<Packet16h>(const Packet16h& a) { - return generic_rsqrt_newton_step<Packet16h, /*Steps=*/1>::run(a, _mm256_rsqrt_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE Packet8h prsqrt<Packet8h>(const Packet8h& a) { - return generic_rsqrt_newton_step<Packet8h, /*Steps=*/1>::run(a, _mm_rsqrt_ph(a)); -} - -// preciprocal - -template <> -EIGEN_STRONG_INLINE Packet32h preciprocal<Packet32h>(const Packet32h& a) { - return generic_reciprocal_newton_step<Packet32h, /*Steps=*/1>::run(a, _mm512_rcp_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE Packet16h preciprocal<Packet16h>(const Packet16h& a) { - return generic_reciprocal_newton_step<Packet16h, /*Steps=*/1>::run(a, _mm256_rcp_ph(a)); -} - -template <> -EIGEN_STRONG_INLINE Packet8h preciprocal<Packet8h>(const Packet8h& a) { - return generic_reciprocal_newton_step<Packet8h, /*Steps=*/1>::run(a, _mm_rcp_ph(a)); -} - -// ptranspose - -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet32h, 32>& a) { - __m512i t[32]; - - EIGEN_UNROLL_LOOP - for (int i = 0; i < 16; i++) { - t[2 * i] = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1])); - t[2 * i + 1] = - _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1])); - } - - __m512i p[32]; - - EIGEN_UNROLL_LOOP - for (int i = 0; i < 8; i++) { - p[4 * i] = _mm512_unpacklo_epi32(t[4 * i], t[4 * i + 2]); - p[4 * i + 1] = _mm512_unpackhi_epi32(t[4 * i], t[4 * i + 2]); - p[4 * i + 2] = _mm512_unpacklo_epi32(t[4 * i + 1], t[4 * i + 3]); - p[4 * i + 3] = _mm512_unpackhi_epi32(t[4 * i + 1], t[4 * i + 3]); - } - - __m512i q[32]; - - EIGEN_UNROLL_LOOP - for (int i = 0; i < 4; i++) { - q[8 * i] = _mm512_unpacklo_epi64(p[8 * i], p[8 * i + 4]); - q[8 * i + 1] = _mm512_unpackhi_epi64(p[8 * i], p[8 * i + 4]); - q[8 * i + 2] = _mm512_unpacklo_epi64(p[8 * i + 1], p[8 * i + 5]); - q[8 * i + 3] = _mm512_unpackhi_epi64(p[8 * i + 1], p[8 * i + 5]); - q[8 * i + 4] = _mm512_unpacklo_epi64(p[8 * i + 2], p[8 * i + 6]); - q[8 * i + 5] = _mm512_unpackhi_epi64(p[8 * i + 2], p[8 * i + 6]); - q[8 * i + 6] = _mm512_unpacklo_epi64(p[8 * i + 3], p[8 * i + 7]); - q[8 * i + 7] = _mm512_unpackhi_epi64(p[8 * i + 3], p[8 * i + 7]); - } - - __m512i f[32]; - -#define PACKET32H_TRANSPOSE_HELPER(X, Y) \ - do { \ - f[Y * 8] = _mm512_inserti32x4(f[Y * 8], _mm512_extracti32x4_epi32(q[X * 8], Y), X); \ - f[Y * 8 + 1] = _mm512_inserti32x4(f[Y * 8 + 1], _mm512_extracti32x4_epi32(q[X * 8 + 1], Y), X); \ - f[Y * 8 + 2] = _mm512_inserti32x4(f[Y * 8 + 2], _mm512_extracti32x4_epi32(q[X * 8 + 2], Y), X); \ - f[Y * 8 + 3] = _mm512_inserti32x4(f[Y * 8 + 3], _mm512_extracti32x4_epi32(q[X * 8 + 3], Y), X); \ - f[Y * 8 + 4] = _mm512_inserti32x4(f[Y * 8 + 4], _mm512_extracti32x4_epi32(q[X * 8 + 4], Y), X); \ - f[Y * 8 + 5] = _mm512_inserti32x4(f[Y * 8 + 5], _mm512_extracti32x4_epi32(q[X * 8 + 5], Y), X); \ - f[Y * 8 + 6] = _mm512_inserti32x4(f[Y * 8 + 6], _mm512_extracti32x4_epi32(q[X * 8 + 6], Y), X); \ - f[Y * 8 + 7] = _mm512_inserti32x4(f[Y * 8 + 7], _mm512_extracti32x4_epi32(q[X * 8 + 7], Y), X); \ - } while (false); - - PACKET32H_TRANSPOSE_HELPER(0, 0); - PACKET32H_TRANSPOSE_HELPER(1, 1); - PACKET32H_TRANSPOSE_HELPER(2, 2); - PACKET32H_TRANSPOSE_HELPER(3, 3); - - PACKET32H_TRANSPOSE_HELPER(1, 0); - PACKET32H_TRANSPOSE_HELPER(2, 0); - PACKET32H_TRANSPOSE_HELPER(3, 0); - PACKET32H_TRANSPOSE_HELPER(2, 1); - PACKET32H_TRANSPOSE_HELPER(3, 1); - PACKET32H_TRANSPOSE_HELPER(3, 2); - - PACKET32H_TRANSPOSE_HELPER(0, 1); - PACKET32H_TRANSPOSE_HELPER(0, 2); - PACKET32H_TRANSPOSE_HELPER(0, 3); - PACKET32H_TRANSPOSE_HELPER(1, 2); - PACKET32H_TRANSPOSE_HELPER(1, 3); - PACKET32H_TRANSPOSE_HELPER(2, 3); - -#undef PACKET32H_TRANSPOSE_HELPER - - EIGEN_UNROLL_LOOP - for (int i = 0; i < 32; i++) { - a.packet[i] = _mm512_castsi512_ph(f[i]); - } -} - -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet32h, 4>& a) { - __m512i p0, p1, p2, p3, t0, t1, t2, t3, a0, a1, a2, a3; - t0 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1])); - t1 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1])); - t2 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3])); - t3 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3])); - - p0 = _mm512_unpacklo_epi32(t0, t2); - p1 = _mm512_unpackhi_epi32(t0, t2); - p2 = _mm512_unpacklo_epi32(t1, t3); - p3 = _mm512_unpackhi_epi32(t1, t3); - - a0 = p0; - a1 = p1; - a2 = p2; - a3 = p3; - - a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p1, 0), 1); - a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p0, 1), 0); - - a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p2, 0), 2); - a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p0, 2), 0); - - a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p3, 0), 3); - a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p0, 3), 0); - - a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p2, 1), 2); - a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p1, 2), 1); - - a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p3, 2), 3); - a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p2, 3), 2); - - a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p3, 1), 3); - a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p1, 3), 1); - - a.packet[0] = _mm512_castsi512_ph(a0); - a.packet[1] = _mm512_castsi512_ph(a1); - a.packet[2] = _mm512_castsi512_ph(a2); - a.packet[3] = _mm512_castsi512_ph(a3); -} - -EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 16>& kernel) { - __m256i a = _mm256_castph_si256(kernel.packet[0]); - __m256i b = _mm256_castph_si256(kernel.packet[1]); - __m256i c = _mm256_castph_si256(kernel.packet[2]); - __m256i d = _mm256_castph_si256(kernel.packet[3]); - __m256i e = _mm256_castph_si256(kernel.packet[4]); - __m256i f = _mm256_castph_si256(kernel.packet[5]); - __m256i g = _mm256_castph_si256(kernel.packet[6]); - __m256i h = _mm256_castph_si256(kernel.packet[7]); - __m256i i = _mm256_castph_si256(kernel.packet[8]); - __m256i j = _mm256_castph_si256(kernel.packet[9]); - __m256i k = _mm256_castph_si256(kernel.packet[10]); - __m256i l = _mm256_castph_si256(kernel.packet[11]); - __m256i m = _mm256_castph_si256(kernel.packet[12]); - __m256i n = _mm256_castph_si256(kernel.packet[13]); - __m256i o = _mm256_castph_si256(kernel.packet[14]); - __m256i p = _mm256_castph_si256(kernel.packet[15]); - - __m256i ab_07 = _mm256_unpacklo_epi16(a, b); - __m256i cd_07 = _mm256_unpacklo_epi16(c, d); - __m256i ef_07 = _mm256_unpacklo_epi16(e, f); - __m256i gh_07 = _mm256_unpacklo_epi16(g, h); - __m256i ij_07 = _mm256_unpacklo_epi16(i, j); - __m256i kl_07 = _mm256_unpacklo_epi16(k, l); - __m256i mn_07 = _mm256_unpacklo_epi16(m, n); - __m256i op_07 = _mm256_unpacklo_epi16(o, p); - - __m256i ab_8f = _mm256_unpackhi_epi16(a, b); - __m256i cd_8f = _mm256_unpackhi_epi16(c, d); - __m256i ef_8f = _mm256_unpackhi_epi16(e, f); - __m256i gh_8f = _mm256_unpackhi_epi16(g, h); - __m256i ij_8f = _mm256_unpackhi_epi16(i, j); - __m256i kl_8f = _mm256_unpackhi_epi16(k, l); - __m256i mn_8f = _mm256_unpackhi_epi16(m, n); - __m256i op_8f = _mm256_unpackhi_epi16(o, p); - - __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07); - __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07); - __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07); - __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07); - __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07); - __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07); - __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07); - __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07); - - __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f); - __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f); - __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f); - __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f); - __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f); - __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f); - __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f); - __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f); - - __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03); - __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03); - __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03); - __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03); - __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47); - __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47); - __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47); - __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47); - __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b); - __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b); - __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b); - __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b); - __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf); - __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf); - __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf); - __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf); - - // NOTE: no unpacklo/hi instr in this case, so using permute instr. - __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20); - __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20); - __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20); - __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20); - __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20); - __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20); - __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20); - __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20); - __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31); - __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31); - __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31); - __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31); - __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31); - __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31); - __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31); - __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31); - - kernel.packet[0] = _mm256_castsi256_ph(a_p_0); - kernel.packet[1] = _mm256_castsi256_ph(a_p_1); - kernel.packet[2] = _mm256_castsi256_ph(a_p_2); - kernel.packet[3] = _mm256_castsi256_ph(a_p_3); - kernel.packet[4] = _mm256_castsi256_ph(a_p_4); - kernel.packet[5] = _mm256_castsi256_ph(a_p_5); - kernel.packet[6] = _mm256_castsi256_ph(a_p_6); - kernel.packet[7] = _mm256_castsi256_ph(a_p_7); - kernel.packet[8] = _mm256_castsi256_ph(a_p_8); - kernel.packet[9] = _mm256_castsi256_ph(a_p_9); - kernel.packet[10] = _mm256_castsi256_ph(a_p_a); - kernel.packet[11] = _mm256_castsi256_ph(a_p_b); - kernel.packet[12] = _mm256_castsi256_ph(a_p_c); - kernel.packet[13] = _mm256_castsi256_ph(a_p_d); - kernel.packet[14] = _mm256_castsi256_ph(a_p_e); - kernel.packet[15] = _mm256_castsi256_ph(a_p_f); -} - -EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 8>& kernel) { - EIGEN_ALIGN64 half in[8][16]; - pstore<half>(in[0], kernel.packet[0]); - pstore<half>(in[1], kernel.packet[1]); - pstore<half>(in[2], kernel.packet[2]); - pstore<half>(in[3], kernel.packet[3]); - pstore<half>(in[4], kernel.packet[4]); - pstore<half>(in[5], kernel.packet[5]); - pstore<half>(in[6], kernel.packet[6]); - pstore<half>(in[7], kernel.packet[7]); - - EIGEN_ALIGN64 half out[8][16]; - - for (int i = 0; i < 8; ++i) { - for (int j = 0; j < 8; ++j) { - out[i][j] = in[j][2 * i]; - } - for (int j = 0; j < 8; ++j) { - out[i][j + 8] = in[j][2 * i + 1]; - } - } - - kernel.packet[0] = pload<Packet16h>(out[0]); - kernel.packet[1] = pload<Packet16h>(out[1]); - kernel.packet[2] = pload<Packet16h>(out[2]); - kernel.packet[3] = pload<Packet16h>(out[3]); - kernel.packet[4] = pload<Packet16h>(out[4]); - kernel.packet[5] = pload<Packet16h>(out[5]); - kernel.packet[6] = pload<Packet16h>(out[6]); - kernel.packet[7] = pload<Packet16h>(out[7]); -} - -EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 4>& kernel) { - EIGEN_ALIGN64 half in[4][16]; - pstore<half>(in[0], kernel.packet[0]); - pstore<half>(in[1], kernel.packet[1]); - pstore<half>(in[2], kernel.packet[2]); - pstore<half>(in[3], kernel.packet[3]); - - EIGEN_ALIGN64 half out[4][16]; - - for (int i = 0; i < 4; ++i) { - for (int j = 0; j < 4; ++j) { - out[i][j] = in[j][4 * i]; - } - for (int j = 0; j < 4; ++j) { - out[i][j + 4] = in[j][4 * i + 1]; - } - for (int j = 0; j < 4; ++j) { - out[i][j + 8] = in[j][4 * i + 2]; - } - for (int j = 0; j < 4; ++j) { - out[i][j + 12] = in[j][4 * i + 3]; - } - } - - kernel.packet[0] = pload<Packet16h>(out[0]); - kernel.packet[1] = pload<Packet16h>(out[1]); - kernel.packet[2] = pload<Packet16h>(out[2]); - kernel.packet[3] = pload<Packet16h>(out[3]); -} - -EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8h, 8>& kernel) { - __m128i a = _mm_castph_si128(kernel.packet[0]); - __m128i b = _mm_castph_si128(kernel.packet[1]); - __m128i c = _mm_castph_si128(kernel.packet[2]); - __m128i d = _mm_castph_si128(kernel.packet[3]); - __m128i e = _mm_castph_si128(kernel.packet[4]); - __m128i f = _mm_castph_si128(kernel.packet[5]); - __m128i g = _mm_castph_si128(kernel.packet[6]); - __m128i h = _mm_castph_si128(kernel.packet[7]); - - __m128i a03b03 = _mm_unpacklo_epi16(a, b); - __m128i c03d03 = _mm_unpacklo_epi16(c, d); - __m128i e03f03 = _mm_unpacklo_epi16(e, f); - __m128i g03h03 = _mm_unpacklo_epi16(g, h); - __m128i a47b47 = _mm_unpackhi_epi16(a, b); - __m128i c47d47 = _mm_unpackhi_epi16(c, d); - __m128i e47f47 = _mm_unpackhi_epi16(e, f); - __m128i g47h47 = _mm_unpackhi_epi16(g, h); - - __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03); - __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03); - __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03); - __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03); - __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47); - __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47); - __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47); - __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47); - - __m128i a0b0c0d0e0f0g0h0 = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01); - __m128i a1b1c1d1e1f1g1h1 = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01); - __m128i a2b2c2d2e2f2g2h2 = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23); - __m128i a3b3c3d3e3f3g3h3 = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23); - __m128i a4b4c4d4e4f4g4h4 = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45); - __m128i a5b5c5d5e5f5g5h5 = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45); - __m128i a6b6c6d6e6f6g6h6 = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67); - __m128i a7b7c7d7e7f7g7h7 = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67); - - kernel.packet[0] = _mm_castsi128_ph(a0b0c0d0e0f0g0h0); - kernel.packet[1] = _mm_castsi128_ph(a1b1c1d1e1f1g1h1); - kernel.packet[2] = _mm_castsi128_ph(a2b2c2d2e2f2g2h2); - kernel.packet[3] = _mm_castsi128_ph(a3b3c3d3e3f3g3h3); - kernel.packet[4] = _mm_castsi128_ph(a4b4c4d4e4f4g4h4); - kernel.packet[5] = _mm_castsi128_ph(a5b5c5d5e5f5g5h5); - kernel.packet[6] = _mm_castsi128_ph(a6b6c6d6e6f6g6h6); - kernel.packet[7] = _mm_castsi128_ph(a7b7c7d7e7f7g7h7); -} - -EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8h, 4>& kernel) { - EIGEN_ALIGN32 Eigen::half in[4][8]; - pstore<Eigen::half>(in[0], kernel.packet[0]); - pstore<Eigen::half>(in[1], kernel.packet[1]); - pstore<Eigen::half>(in[2], kernel.packet[2]); - pstore<Eigen::half>(in[3], kernel.packet[3]); - - EIGEN_ALIGN32 Eigen::half out[4][8]; - - for (int i = 0; i < 4; ++i) { - for (int j = 0; j < 4; ++j) { - out[i][j] = in[j][2 * i]; - } - for (int j = 0; j < 4; ++j) { - out[i][j + 4] = in[j][2 * i + 1]; - } - } - - kernel.packet[0] = pload<Packet8h>(out[0]); - kernel.packet[1] = pload<Packet8h>(out[1]); - kernel.packet[2] = pload<Packet8h>(out[2]); - kernel.packet[3] = pload<Packet8h>(out[3]); -} - -// preverse - -template <> -EIGEN_STRONG_INLINE Packet32h preverse(const Packet32h& a) { - return _mm512_permutexvar_ph(_mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31), - a); -} - -template <> -EIGEN_STRONG_INLINE Packet16h preverse(const Packet16h& a) { - __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); - return _mm256_castsi256_ph(_mm256_insertf128_si256( - _mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castph_si256(a), 1), m)), - _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castph_si256(a), 0), m), 1)); -} - -template <> -EIGEN_STRONG_INLINE Packet8h preverse(const Packet8h& a) { - __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); - return _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(a), m)); -} - -// pscatter - -template <> -EIGEN_STRONG_INLINE void pscatter<half, Packet32h>(half* to, const Packet32h& from, Index stride) { - EIGEN_ALIGN64 half aux[32]; - pstore(aux, from); - - EIGEN_UNROLL_LOOP - for (int i = 0; i < 32; i++) { - to[stride * i] = aux[i]; - } -} -template <> -EIGEN_STRONG_INLINE void pscatter<half, Packet16h>(half* to, const Packet16h& from, Index stride) { - EIGEN_ALIGN64 half aux[16]; - pstore(aux, from); - to[stride * 0] = aux[0]; - to[stride * 1] = aux[1]; - to[stride * 2] = aux[2]; - to[stride * 3] = aux[3]; - to[stride * 4] = aux[4]; - to[stride * 5] = aux[5]; - to[stride * 6] = aux[6]; - to[stride * 7] = aux[7]; - to[stride * 8] = aux[8]; - to[stride * 9] = aux[9]; - to[stride * 10] = aux[10]; - to[stride * 11] = aux[11]; - to[stride * 12] = aux[12]; - to[stride * 13] = aux[13]; - to[stride * 14] = aux[14]; - to[stride * 15] = aux[15]; -} - -template <> -EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet8h>(Eigen::half* to, const Packet8h& from, Index stride) { - EIGEN_ALIGN32 Eigen::half aux[8]; - pstore(aux, from); - to[stride * 0] = aux[0]; - to[stride * 1] = aux[1]; - to[stride * 2] = aux[2]; - to[stride * 3] = aux[3]; - to[stride * 4] = aux[4]; - to[stride * 5] = aux[5]; - to[stride * 6] = aux[6]; - to[stride * 7] = aux[7]; -} - -// pgather - -template <> -EIGEN_STRONG_INLINE Packet32h pgather<Eigen::half, Packet32h>(const Eigen::half* from, Index stride) { - return _mm512_set_ph(from[31 * stride].x, from[30 * stride].x, from[29 * stride].x, from[28 * stride].x, - from[27 * stride].x, from[26 * stride].x, from[25 * stride].x, from[24 * stride].x, - from[23 * stride].x, from[22 * stride].x, from[21 * stride].x, from[20 * stride].x, - from[19 * stride].x, from[18 * stride].x, from[17 * stride].x, from[16 * stride].x, - from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x, - from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x, - from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, - from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x); -} - -template <> -EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(const Eigen::half* from, Index stride) { - return _mm256_set_ph(from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x, - from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x, - from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, - from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x); -} - -template <> -EIGEN_STRONG_INLINE Packet8h pgather<Eigen::half, Packet8h>(const Eigen::half* from, Index stride) { - return _mm_set_ph(from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, from[3 * stride].x, - from[2 * stride].x, from[1 * stride].x, from[0 * stride].x); -} - -} // end namespace internal -} // end namespace Eigen - -#endif // EIGEN_PACKET_MATH_FP16_AVX512_H +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2025 The Eigen Authors. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_PACKET_MATH_FP16_AVX512_H +#define EIGEN_PACKET_MATH_FP16_AVX512_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +typedef __m512h Packet32h; +typedef __m256h Packet16h; +typedef __m128h Packet8h; + +template <> +struct is_arithmetic<Packet8h> { + enum { value = true }; +}; + +template <> +struct packet_traits<half> : default_packet_traits { + typedef Packet32h type; + typedef Packet16h half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 32, + + HasCmp = 1, + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasAbs = 1, + HasAbs2 = 0, + HasMin = 1, + HasMax = 1, + HasConj = 1, + HasSetLinear = 0, + HasLog = 1, + HasLog1p = 1, + HasExp = 1, + HasExpm1 = 1, + HasSqrt = 1, + HasRsqrt = 1, + // These ones should be implemented in future + HasBessel = 0, + HasNdtri = 0, + HasSin = EIGEN_FAST_MATH, + HasCos = EIGEN_FAST_MATH, + HasTanh = EIGEN_FAST_MATH, + HasErf = 0, // EIGEN_FAST_MATH, + HasBlend = 0 + }; +}; + +template <> +struct unpacket_traits<Packet32h> { + typedef Eigen::half type; + typedef Packet16h half; + typedef Packet32s integer_packet; + enum { + size = 32, + alignment = Aligned64, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +struct unpacket_traits<Packet16h> { + typedef Eigen::half type; + typedef Packet8h half; + typedef Packet16s integer_packet; + enum { + size = 16, + alignment = Aligned32, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +struct unpacket_traits<Packet8h> { + typedef Eigen::half type; + typedef Packet8h half; + typedef Packet8s integer_packet; + enum { + size = 8, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +// Conversions + +EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) { return _mm512_cvtxph_ps(a); } + +EIGEN_STRONG_INLINE Packet8f half2float(const Packet8h& a) { return _mm256_cvtxph_ps(a); } + +EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) { return _mm512_cvtxps_ph(a); } + +EIGEN_STRONG_INLINE Packet8h float2half(const Packet8f& a) { return _mm256_cvtxps_ph(a); } + +// Memory functions + +// pset1 + +template <> +EIGEN_STRONG_INLINE Packet32h pset1<Packet32h>(const Eigen::half& from) { + return _mm512_set1_ph(from.x); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(const Eigen::half& from) { + return _mm256_set1_ph(from.x); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pset1<Packet8h>(const Eigen::half& from) { + return _mm_set1_ph(from.x); +} + +template <> +EIGEN_STRONG_INLINE Packet32h pzero(const Packet32h& /*a*/) { + return _mm512_setzero_ph(); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pzero(const Packet16h& /*a*/) { + return _mm256_setzero_ph(); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pzero(const Packet8h& /*a*/) { + return _mm_setzero_ph(); +} + +// pset1frombits +template <> +EIGEN_STRONG_INLINE Packet32h pset1frombits<Packet32h>(unsigned short from) { + return _mm512_castsi512_ph(_mm512_set1_epi16(from)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pset1frombits<Packet16h>(unsigned short from) { + return _mm256_castsi256_ph(_mm256_set1_epi16(from)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pset1frombits<Packet8h>(unsigned short from) { + return _mm_castsi128_ph(_mm_set1_epi16(from)); +} + +// pfirst + +template <> +EIGEN_STRONG_INLINE Eigen::half pfirst<Packet32h>(const Packet32h& from) { + return Eigen::half(_mm512_cvtsh_h(from)); +} + +template <> +EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(const Packet16h& from) { + return Eigen::half(_mm256_cvtsh_h(from)); +} + +template <> +EIGEN_STRONG_INLINE Eigen::half pfirst<Packet8h>(const Packet8h& from) { + return Eigen::half(_mm_cvtsh_h(from)); +} + +// pload + +template <> +EIGEN_STRONG_INLINE Packet32h pload<Packet32h>(const Eigen::half* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ph(from); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(const Eigen::half* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_ph(from); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pload<Packet8h>(const Eigen::half* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ph(from); +} + +// ploadu + +template <> +EIGEN_STRONG_INLINE Packet32h ploadu<Packet32h>(const Eigen::half* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ph(from); +} + +template <> +EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(const Eigen::half* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_ph(from); +} + +template <> +EIGEN_STRONG_INLINE Packet8h ploadu<Packet8h>(const Eigen::half* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_ph(from); +} + +// pstore + +template <> +EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet32h& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ph(to, from); +} + +template <> +EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet16h& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm256_store_ph(to, from); +} + +template <> +EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet8h& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm_store_ph(to, from); +} + +// pstoreu + +template <> +EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet32h& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ph(to, from); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet16h& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_ph(to, from); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet8h& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_ph(to, from); +} + +// ploaddup +template <> +EIGEN_STRONG_INLINE Packet32h ploaddup<Packet32h>(const Eigen::half* from) { + __m512h a = _mm512_castph256_ph512(_mm256_loadu_ph(from)); + return _mm512_permutexvar_ph(_mm512_set_epi16(15, 15, 14, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, 6, 6, + 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0), + a); +} + +template <> +EIGEN_STRONG_INLINE Packet16h ploaddup<Packet16h>(const Eigen::half* from) { + __m256h a = _mm256_castph128_ph256(_mm_loadu_ph(from)); + return _mm256_permutexvar_ph(_mm256_set_epi16(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0), a); +} + +template <> +EIGEN_STRONG_INLINE Packet8h ploaddup<Packet8h>(const Eigen::half* from) { + return _mm_set_ph(from[3].x, from[3].x, from[2].x, from[2].x, from[1].x, from[1].x, from[0].x, from[0].x); +} + +// ploadquad +template <> +EIGEN_STRONG_INLINE Packet32h ploadquad<Packet32h>(const Eigen::half* from) { + __m512h a = _mm512_castph128_ph512(_mm_loadu_ph(from)); + return _mm512_permutexvar_ph( + _mm512_set_epi16(7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0), + a); +} + +template <> +EIGEN_STRONG_INLINE Packet16h ploadquad<Packet16h>(const Eigen::half* from) { + return _mm256_set_ph(from[3].x, from[3].x, from[3].x, from[3].x, from[2].x, from[2].x, from[2].x, from[2].x, + from[1].x, from[1].x, from[1].x, from[1].x, from[0].x, from[0].x, from[0].x, from[0].x); +} + +template <> +EIGEN_STRONG_INLINE Packet8h ploadquad<Packet8h>(const Eigen::half* from) { + return _mm_set_ph(from[1].x, from[1].x, from[1].x, from[1].x, from[0].x, from[0].x, from[0].x, from[0].x); +} + +// pabs + +template <> +EIGEN_STRONG_INLINE Packet32h pabs<Packet32h>(const Packet32h& a) { + return _mm512_abs_ph(a); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pabs<Packet16h>(const Packet16h& a) { + return _mm256_abs_ph(a); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pabs<Packet8h>(const Packet8h& a) { + return _mm_abs_ph(a); +} + +// psignbit + +template <> +EIGEN_STRONG_INLINE Packet32h psignbit<Packet32h>(const Packet32h& a) { + return _mm512_castsi512_ph(_mm512_srai_epi16(_mm512_castph_si512(a), 15)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h psignbit<Packet16h>(const Packet16h& a) { + return _mm256_castsi256_ph(_mm256_srai_epi16(_mm256_castph_si256(a), 15)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h psignbit<Packet8h>(const Packet8h& a) { + return _mm_castsi128_ph(_mm_srai_epi16(_mm_castph_si128(a), 15)); +} + +// pmin + +template <> +EIGEN_STRONG_INLINE Packet32h pmin<Packet32h>(const Packet32h& a, const Packet32h& b) { + return _mm512_min_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmin<Packet16h>(const Packet16h& a, const Packet16h& b) { + return _mm256_min_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmin<Packet8h>(const Packet8h& a, const Packet8h& b) { + return _mm_min_ph(a, b); +} + +// pmax + +template <> +EIGEN_STRONG_INLINE Packet32h pmax<Packet32h>(const Packet32h& a, const Packet32h& b) { + return _mm512_max_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmax<Packet16h>(const Packet16h& a, const Packet16h& b) { + return _mm256_max_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmax<Packet8h>(const Packet8h& a, const Packet8h& b) { + return _mm_max_ph(a, b); +} + +// plset +template <> +EIGEN_STRONG_INLINE Packet32h plset<Packet32h>(const half& a) { + return _mm512_add_ph(pset1<Packet32h>(a), _mm512_set_ph(31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, + 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h plset<Packet16h>(const half& a) { + return _mm256_add_ph(pset1<Packet16h>(a), _mm256_set_ph(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h plset<Packet8h>(const half& a) { + return _mm_add_ph(pset1<Packet8h>(a), _mm_set_ph(7, 6, 5, 4, 3, 2, 1, 0)); +} + +// por + +template <> +EIGEN_STRONG_INLINE Packet32h por(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_or_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a, const Packet16h& b) { + return _mm256_castsi256_ph(_mm256_or_si256(_mm256_castph_si256(a), _mm256_castph_si256(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h por(const Packet8h& a, const Packet8h& b) { + return _mm_castsi128_ph(_mm_or_si128(_mm_castph_si128(a), _mm_castph_si128(b))); +} + +// pxor + +template <> +EIGEN_STRONG_INLINE Packet32h pxor(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_xor_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a, const Packet16h& b) { + return _mm256_castsi256_ph(_mm256_xor_si256(_mm256_castph_si256(a), _mm256_castph_si256(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pxor(const Packet8h& a, const Packet8h& b) { + return _mm_castsi128_ph(_mm_xor_si128(_mm_castph_si128(a), _mm_castph_si128(b))); +} + +// pand + +template <> +EIGEN_STRONG_INLINE Packet32h pand(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_and_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a, const Packet16h& b) { + return _mm256_castsi256_ph(_mm256_and_si256(_mm256_castph_si256(a), _mm256_castph_si256(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pand(const Packet8h& a, const Packet8h& b) { + return _mm_castsi128_ph(_mm_and_si128(_mm_castph_si128(a), _mm_castph_si128(b))); +} + +// pandnot + +template <> +EIGEN_STRONG_INLINE Packet32h pandnot(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_andnot_si512(_mm512_castph_si512(b), _mm512_castph_si512(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a, const Packet16h& b) { + return _mm256_castsi256_ph(_mm256_andnot_si256(_mm256_castph_si256(b), _mm256_castph_si256(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pandnot(const Packet8h& a, const Packet8h& b) { + return _mm_castsi128_ph(_mm_andnot_si128(_mm_castph_si128(b), _mm_castph_si128(a))); +} + +// pselect + +template <> +EIGEN_DEVICE_FUNC inline Packet32h pselect(const Packet32h& mask, const Packet32h& a, const Packet32h& b) { + __mmask32 mask32 = _mm512_cmp_epi16_mask(_mm512_castph_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ); + return _mm512_mask_blend_ph(mask32, a, b); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet16h pselect(const Packet16h& mask, const Packet16h& a, const Packet16h& b) { + __mmask16 mask16 = _mm256_cmp_epi16_mask(_mm256_castph_si256(mask), _mm256_setzero_si256(), _MM_CMPINT_EQ); + return _mm256_mask_blend_ph(mask16, a, b); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet8h pselect(const Packet8h& mask, const Packet8h& a, const Packet8h& b) { + __mmask8 mask8 = _mm_cmp_epi16_mask(_mm_castph_si128(mask), _mm_setzero_si128(), _MM_CMPINT_EQ); + return _mm_mask_blend_ph(mask8, a, b); +} + +// pcmp_eq + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_eq(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_EQ_OQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a, const Packet16h& b) { + __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_EQ_OQ); + return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pcmp_eq(const Packet8h& a, const Packet8h& b) { + __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_EQ_OQ); + return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +// pcmp_le + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_le(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LE_OQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pcmp_le(const Packet16h& a, const Packet16h& b) { + __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_LE_OQ); + return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pcmp_le(const Packet8h& a, const Packet8h& b) { + __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_LE_OQ); + return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +// pcmp_lt + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_lt(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LT_OQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pcmp_lt(const Packet16h& a, const Packet16h& b) { + __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_LT_OQ); + return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pcmp_lt(const Packet8h& a, const Packet8h& b) { + __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_LT_OQ); + return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +// pcmp_lt_or_nan + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_lt_or_nan(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_NGE_UQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi16(0), mask, static_cast<short>(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pcmp_lt_or_nan(const Packet16h& a, const Packet16h& b) { + __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_NGE_UQ); + return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pcmp_lt_or_nan(const Packet8h& a, const Packet8h& b) { + __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_NGE_UQ); + return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast<short>(0xffffu))); +} + +// padd + +template <> +EIGEN_STRONG_INLINE Packet32h padd<Packet32h>(const Packet32h& a, const Packet32h& b) { + return _mm512_add_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) { + return _mm256_add_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h padd<Packet8h>(const Packet8h& a, const Packet8h& b) { + return _mm_add_ph(a, b); +} + +// psub + +template <> +EIGEN_STRONG_INLINE Packet32h psub<Packet32h>(const Packet32h& a, const Packet32h& b) { + return _mm512_sub_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h psub<Packet16h>(const Packet16h& a, const Packet16h& b) { + return _mm256_sub_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h psub<Packet8h>(const Packet8h& a, const Packet8h& b) { + return _mm_sub_ph(a, b); +} + +// pmul + +template <> +EIGEN_STRONG_INLINE Packet32h pmul<Packet32h>(const Packet32h& a, const Packet32h& b) { + return _mm512_mul_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) { + return _mm256_mul_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmul<Packet8h>(const Packet8h& a, const Packet8h& b) { + return _mm_mul_ph(a, b); +} + +// pdiv + +template <> +EIGEN_STRONG_INLINE Packet32h pdiv<Packet32h>(const Packet32h& a, const Packet32h& b) { + return _mm512_div_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pdiv<Packet16h>(const Packet16h& a, const Packet16h& b) { + return _mm256_div_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pdiv<Packet8h>(const Packet8h& a, const Packet8h& b) { + return _mm_div_ph(a, b); + ; +} + +// pround + +template <> +EIGEN_STRONG_INLINE Packet32h pround<Packet32h>(const Packet32h& a) { + // Work-around for default std::round rounding mode. + + // Mask for the sign bit. + const Packet32h signMask = + pset1frombits<Packet32h>(static_cast<numext::uint16_t>(static_cast<std::uint16_t>(0x8000u))); + // The largest half-precision float less than 0.5. + const Packet32h prev0dot5 = pset1frombits<Packet32h>(static_cast<numext::uint16_t>(0x37FFu)); + + return _mm512_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pround<Packet16h>(const Packet16h& a) { + // Work-around for default std::round rounding mode. + + // Mask for the sign bit. + const Packet16h signMask = + pset1frombits<Packet16h>(static_cast<numext::uint16_t>(static_cast<std::uint16_t>(0x8000u))); + // The largest half-precision float less than 0.5. + const Packet16h prev0dot5 = pset1frombits<Packet16h>(static_cast<numext::uint16_t>(0x37FFu)); + + return _mm256_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pround<Packet8h>(const Packet8h& a) { + // Work-around for default std::round rounding mode. + + // Mask for the sign bit. + const Packet8h signMask = pset1frombits<Packet8h>(static_cast<numext::uint16_t>(static_cast<std::uint16_t>(0x8000u))); + // The largest half-precision float less than 0.5. + const Packet8h prev0dot5 = pset1frombits<Packet8h>(static_cast<numext::uint16_t>(0x37FFu)); + + return _mm_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO); +} + +// print + +template <> +EIGEN_STRONG_INLINE Packet32h print<Packet32h>(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION); +} + +template <> +EIGEN_STRONG_INLINE Packet16h print<Packet16h>(const Packet16h& a) { + return _mm256_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION); +} + +template <> +EIGEN_STRONG_INLINE Packet8h print<Packet8h>(const Packet8h& a) { + return _mm_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION); +} + +// pceil + +template <> +EIGEN_STRONG_INLINE Packet32h pceil<Packet32h>(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_TO_POS_INF); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pceil<Packet16h>(const Packet16h& a) { + return _mm256_roundscale_ph(a, _MM_FROUND_TO_POS_INF); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pceil<Packet8h>(const Packet8h& a) { + return _mm_roundscale_ph(a, _MM_FROUND_TO_POS_INF); +} + +// pfloor + +template <> +EIGEN_STRONG_INLINE Packet32h pfloor<Packet32h>(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_TO_NEG_INF); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pfloor<Packet16h>(const Packet16h& a) { + return _mm256_roundscale_ph(a, _MM_FROUND_TO_NEG_INF); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pfloor<Packet8h>(const Packet8h& a) { + return _mm_roundscale_ph(a, _MM_FROUND_TO_NEG_INF); +} + +// ptrunc + +template <> +EIGEN_STRONG_INLINE Packet32h ptrunc<Packet32h>(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_TO_ZERO); +} + +template <> +EIGEN_STRONG_INLINE Packet16h ptrunc<Packet16h>(const Packet16h& a) { + return _mm256_roundscale_ph(a, _MM_FROUND_TO_ZERO); +} + +template <> +EIGEN_STRONG_INLINE Packet8h ptrunc<Packet8h>(const Packet8h& a) { + return _mm_roundscale_ph(a, _MM_FROUND_TO_ZERO); +} + +// predux +template <> +EIGEN_STRONG_INLINE half predux<Packet32h>(const Packet32h& a) { + return half(_mm512_reduce_add_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux<Packet16h>(const Packet16h& a) { + return half(_mm256_reduce_add_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux<Packet8h>(const Packet8h& a) { + return half(_mm_reduce_add_ph(a)); +} + +// predux_half_dowto4 +template <> +EIGEN_STRONG_INLINE Packet16h predux_half_dowto4<Packet32h>(const Packet32h& a) { + const __m512i bits = _mm512_castph_si512(a); + Packet16h lo = _mm256_castsi256_ph(_mm512_castsi512_si256(bits)); + Packet16h hi = _mm256_castsi256_ph(_mm512_extracti64x4_epi64(bits, 1)); + return padd(lo, hi); +} + +template <> +EIGEN_STRONG_INLINE Packet8h predux_half_dowto4<Packet16h>(const Packet16h& a) { + Packet8h lo = _mm_castsi128_ph(_mm256_castsi256_si128(_mm256_castph_si256(a))); + Packet8h hi = _mm_castps_ph(_mm256_extractf128_ps(_mm256_castph_ps(a), 1)); + return padd(lo, hi); +} + +// predux_max + +template <> +EIGEN_STRONG_INLINE half predux_max<Packet32h>(const Packet32h& a) { + return half(_mm512_reduce_max_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_max<Packet16h>(const Packet16h& a) { + return half(_mm256_reduce_max_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_max<Packet8h>(const Packet8h& a) { + return half(_mm_reduce_max_ph(a)); +} + +// predux_min + +template <> +EIGEN_STRONG_INLINE half predux_min<Packet32h>(const Packet32h& a) { + return half(_mm512_reduce_min_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_min<Packet16h>(const Packet16h& a) { + return half(_mm256_reduce_min_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_min<Packet8h>(const Packet8h& a) { + return half(_mm_reduce_min_ph(a)); +} + +// predux_mul + +template <> +EIGEN_STRONG_INLINE half predux_mul<Packet32h>(const Packet32h& a) { + return half(_mm512_reduce_mul_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_mul<Packet16h>(const Packet16h& a) { + return half(_mm256_reduce_mul_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_mul<Packet8h>(const Packet8h& a) { + return half(_mm_reduce_mul_ph(a)); +} + +#ifdef EIGEN_VECTORIZE_FMA + +// pmadd + +template <> +EIGEN_STRONG_INLINE Packet32h pmadd(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fmadd_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmadd(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_fmadd_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmadd(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_fmadd_ph(a, b, c); +} + +// pmsub + +template <> +EIGEN_STRONG_INLINE Packet32h pmsub(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fmsub_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmsub(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_fmsub_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmsub(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_fmsub_ph(a, b, c); +} + +// pnmadd + +template <> +EIGEN_STRONG_INLINE Packet32h pnmadd(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fnmadd_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pnmadd(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_fnmadd_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pnmadd(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_fnmadd_ph(a, b, c); +} + +// pnmsub + +template <> +EIGEN_STRONG_INLINE Packet32h pnmsub(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fnmsub_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pnmsub(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_fnmsub_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pnmsub(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_fnmsub_ph(a, b, c); +} + +#endif + +// pnegate + +template <> +EIGEN_STRONG_INLINE Packet32h pnegate<Packet32h>(const Packet32h& a) { + return _mm512_castsi512_ph( + _mm512_xor_si512(_mm512_castph_si512(a), _mm512_set1_epi16(static_cast<std::uint16_t>(0x8000u)))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pnegate<Packet16h>(const Packet16h& a) { + return _mm256_castsi256_ph( + _mm256_xor_si256(_mm256_castph_si256(a), _mm256_set1_epi16(static_cast<std::uint16_t>(0x8000u)))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pnegate<Packet8h>(const Packet8h& a) { + return _mm_castsi128_ph(_mm_xor_si128(_mm_castph_si128(a), _mm_set1_epi16(static_cast<std::uint16_t>(0x8000u)))); +} + +// pconj + +// Nothing, packets are real. + +// psqrt + +template <> +EIGEN_STRONG_INLINE Packet32h psqrt<Packet32h>(const Packet32h& a) { + return generic_sqrt_newton_step<Packet32h>::run(a, _mm512_rsqrt_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h psqrt<Packet16h>(const Packet16h& a) { + return generic_sqrt_newton_step<Packet16h>::run(a, _mm256_rsqrt_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h psqrt<Packet8h>(const Packet8h& a) { + return generic_sqrt_newton_step<Packet8h>::run(a, _mm_rsqrt_ph(a)); +} + +// prsqrt + +template <> +EIGEN_STRONG_INLINE Packet32h prsqrt<Packet32h>(const Packet32h& a) { + return generic_rsqrt_newton_step<Packet32h, /*Steps=*/1>::run(a, _mm512_rsqrt_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h prsqrt<Packet16h>(const Packet16h& a) { + return generic_rsqrt_newton_step<Packet16h, /*Steps=*/1>::run(a, _mm256_rsqrt_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h prsqrt<Packet8h>(const Packet8h& a) { + return generic_rsqrt_newton_step<Packet8h, /*Steps=*/1>::run(a, _mm_rsqrt_ph(a)); +} + +// preciprocal + +template <> +EIGEN_STRONG_INLINE Packet32h preciprocal<Packet32h>(const Packet32h& a) { + return generic_reciprocal_newton_step<Packet32h, /*Steps=*/1>::run(a, _mm512_rcp_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h preciprocal<Packet16h>(const Packet16h& a) { + return generic_reciprocal_newton_step<Packet16h, /*Steps=*/1>::run(a, _mm256_rcp_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h preciprocal<Packet8h>(const Packet8h& a) { + return generic_reciprocal_newton_step<Packet8h, /*Steps=*/1>::run(a, _mm_rcp_ph(a)); +} + +// ptranspose + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet32h, 32>& a) { + __m512i t[32]; + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 16; i++) { + t[2 * i] = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1])); + t[2 * i + 1] = + _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1])); + } + + __m512i p[32]; + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 8; i++) { + p[4 * i] = _mm512_unpacklo_epi32(t[4 * i], t[4 * i + 2]); + p[4 * i + 1] = _mm512_unpackhi_epi32(t[4 * i], t[4 * i + 2]); + p[4 * i + 2] = _mm512_unpacklo_epi32(t[4 * i + 1], t[4 * i + 3]); + p[4 * i + 3] = _mm512_unpackhi_epi32(t[4 * i + 1], t[4 * i + 3]); + } + + __m512i q[32]; + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 4; i++) { + q[8 * i] = _mm512_unpacklo_epi64(p[8 * i], p[8 * i + 4]); + q[8 * i + 1] = _mm512_unpackhi_epi64(p[8 * i], p[8 * i + 4]); + q[8 * i + 2] = _mm512_unpacklo_epi64(p[8 * i + 1], p[8 * i + 5]); + q[8 * i + 3] = _mm512_unpackhi_epi64(p[8 * i + 1], p[8 * i + 5]); + q[8 * i + 4] = _mm512_unpacklo_epi64(p[8 * i + 2], p[8 * i + 6]); + q[8 * i + 5] = _mm512_unpackhi_epi64(p[8 * i + 2], p[8 * i + 6]); + q[8 * i + 6] = _mm512_unpacklo_epi64(p[8 * i + 3], p[8 * i + 7]); + q[8 * i + 7] = _mm512_unpackhi_epi64(p[8 * i + 3], p[8 * i + 7]); + } + + __m512i f[32]; + +#define PACKET32H_TRANSPOSE_HELPER(X, Y) \ + do { \ + f[Y * 8] = _mm512_inserti32x4(f[Y * 8], _mm512_extracti32x4_epi32(q[X * 8], Y), X); \ + f[Y * 8 + 1] = _mm512_inserti32x4(f[Y * 8 + 1], _mm512_extracti32x4_epi32(q[X * 8 + 1], Y), X); \ + f[Y * 8 + 2] = _mm512_inserti32x4(f[Y * 8 + 2], _mm512_extracti32x4_epi32(q[X * 8 + 2], Y), X); \ + f[Y * 8 + 3] = _mm512_inserti32x4(f[Y * 8 + 3], _mm512_extracti32x4_epi32(q[X * 8 + 3], Y), X); \ + f[Y * 8 + 4] = _mm512_inserti32x4(f[Y * 8 + 4], _mm512_extracti32x4_epi32(q[X * 8 + 4], Y), X); \ + f[Y * 8 + 5] = _mm512_inserti32x4(f[Y * 8 + 5], _mm512_extracti32x4_epi32(q[X * 8 + 5], Y), X); \ + f[Y * 8 + 6] = _mm512_inserti32x4(f[Y * 8 + 6], _mm512_extracti32x4_epi32(q[X * 8 + 6], Y), X); \ + f[Y * 8 + 7] = _mm512_inserti32x4(f[Y * 8 + 7], _mm512_extracti32x4_epi32(q[X * 8 + 7], Y), X); \ + } while (false); + + PACKET32H_TRANSPOSE_HELPER(0, 0); + PACKET32H_TRANSPOSE_HELPER(1, 1); + PACKET32H_TRANSPOSE_HELPER(2, 2); + PACKET32H_TRANSPOSE_HELPER(3, 3); + + PACKET32H_TRANSPOSE_HELPER(1, 0); + PACKET32H_TRANSPOSE_HELPER(2, 0); + PACKET32H_TRANSPOSE_HELPER(3, 0); + PACKET32H_TRANSPOSE_HELPER(2, 1); + PACKET32H_TRANSPOSE_HELPER(3, 1); + PACKET32H_TRANSPOSE_HELPER(3, 2); + + PACKET32H_TRANSPOSE_HELPER(0, 1); + PACKET32H_TRANSPOSE_HELPER(0, 2); + PACKET32H_TRANSPOSE_HELPER(0, 3); + PACKET32H_TRANSPOSE_HELPER(1, 2); + PACKET32H_TRANSPOSE_HELPER(1, 3); + PACKET32H_TRANSPOSE_HELPER(2, 3); + +#undef PACKET32H_TRANSPOSE_HELPER + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 32; i++) { + a.packet[i] = _mm512_castsi512_ph(f[i]); + } +} + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet32h, 4>& a) { + __m512i p0, p1, p2, p3, t0, t1, t2, t3, a0, a1, a2, a3; + t0 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1])); + t1 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1])); + t2 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3])); + t3 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3])); + + p0 = _mm512_unpacklo_epi32(t0, t2); + p1 = _mm512_unpackhi_epi32(t0, t2); + p2 = _mm512_unpacklo_epi32(t1, t3); + p3 = _mm512_unpackhi_epi32(t1, t3); + + a0 = p0; + a1 = p1; + a2 = p2; + a3 = p3; + + a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p1, 0), 1); + a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p0, 1), 0); + + a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p2, 0), 2); + a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p0, 2), 0); + + a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p3, 0), 3); + a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p0, 3), 0); + + a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p2, 1), 2); + a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p1, 2), 1); + + a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p3, 2), 3); + a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p2, 3), 2); + + a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p3, 1), 3); + a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p1, 3), 1); + + a.packet[0] = _mm512_castsi512_ph(a0); + a.packet[1] = _mm512_castsi512_ph(a1); + a.packet[2] = _mm512_castsi512_ph(a2); + a.packet[3] = _mm512_castsi512_ph(a3); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 16>& kernel) { + __m256i a = _mm256_castph_si256(kernel.packet[0]); + __m256i b = _mm256_castph_si256(kernel.packet[1]); + __m256i c = _mm256_castph_si256(kernel.packet[2]); + __m256i d = _mm256_castph_si256(kernel.packet[3]); + __m256i e = _mm256_castph_si256(kernel.packet[4]); + __m256i f = _mm256_castph_si256(kernel.packet[5]); + __m256i g = _mm256_castph_si256(kernel.packet[6]); + __m256i h = _mm256_castph_si256(kernel.packet[7]); + __m256i i = _mm256_castph_si256(kernel.packet[8]); + __m256i j = _mm256_castph_si256(kernel.packet[9]); + __m256i k = _mm256_castph_si256(kernel.packet[10]); + __m256i l = _mm256_castph_si256(kernel.packet[11]); + __m256i m = _mm256_castph_si256(kernel.packet[12]); + __m256i n = _mm256_castph_si256(kernel.packet[13]); + __m256i o = _mm256_castph_si256(kernel.packet[14]); + __m256i p = _mm256_castph_si256(kernel.packet[15]); + + __m256i ab_07 = _mm256_unpacklo_epi16(a, b); + __m256i cd_07 = _mm256_unpacklo_epi16(c, d); + __m256i ef_07 = _mm256_unpacklo_epi16(e, f); + __m256i gh_07 = _mm256_unpacklo_epi16(g, h); + __m256i ij_07 = _mm256_unpacklo_epi16(i, j); + __m256i kl_07 = _mm256_unpacklo_epi16(k, l); + __m256i mn_07 = _mm256_unpacklo_epi16(m, n); + __m256i op_07 = _mm256_unpacklo_epi16(o, p); + + __m256i ab_8f = _mm256_unpackhi_epi16(a, b); + __m256i cd_8f = _mm256_unpackhi_epi16(c, d); + __m256i ef_8f = _mm256_unpackhi_epi16(e, f); + __m256i gh_8f = _mm256_unpackhi_epi16(g, h); + __m256i ij_8f = _mm256_unpackhi_epi16(i, j); + __m256i kl_8f = _mm256_unpackhi_epi16(k, l); + __m256i mn_8f = _mm256_unpackhi_epi16(m, n); + __m256i op_8f = _mm256_unpackhi_epi16(o, p); + + __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07); + __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07); + __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07); + __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07); + __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07); + __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07); + __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07); + __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07); + + __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f); + __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f); + __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f); + __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f); + __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f); + __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f); + __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f); + __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f); + + __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03); + __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03); + __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03); + __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03); + __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47); + __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47); + __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47); + __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47); + __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b); + __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b); + __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b); + __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b); + __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf); + __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf); + __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf); + __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf); + + // NOTE: no unpacklo/hi instr in this case, so using permute instr. + __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20); + __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20); + __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20); + __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20); + __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20); + __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20); + __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20); + __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20); + __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31); + __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31); + __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31); + __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31); + __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31); + __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31); + __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31); + __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31); + + kernel.packet[0] = _mm256_castsi256_ph(a_p_0); + kernel.packet[1] = _mm256_castsi256_ph(a_p_1); + kernel.packet[2] = _mm256_castsi256_ph(a_p_2); + kernel.packet[3] = _mm256_castsi256_ph(a_p_3); + kernel.packet[4] = _mm256_castsi256_ph(a_p_4); + kernel.packet[5] = _mm256_castsi256_ph(a_p_5); + kernel.packet[6] = _mm256_castsi256_ph(a_p_6); + kernel.packet[7] = _mm256_castsi256_ph(a_p_7); + kernel.packet[8] = _mm256_castsi256_ph(a_p_8); + kernel.packet[9] = _mm256_castsi256_ph(a_p_9); + kernel.packet[10] = _mm256_castsi256_ph(a_p_a); + kernel.packet[11] = _mm256_castsi256_ph(a_p_b); + kernel.packet[12] = _mm256_castsi256_ph(a_p_c); + kernel.packet[13] = _mm256_castsi256_ph(a_p_d); + kernel.packet[14] = _mm256_castsi256_ph(a_p_e); + kernel.packet[15] = _mm256_castsi256_ph(a_p_f); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 8>& kernel) { + EIGEN_ALIGN64 half in[8][16]; + pstore<half>(in[0], kernel.packet[0]); + pstore<half>(in[1], kernel.packet[1]); + pstore<half>(in[2], kernel.packet[2]); + pstore<half>(in[3], kernel.packet[3]); + pstore<half>(in[4], kernel.packet[4]); + pstore<half>(in[5], kernel.packet[5]); + pstore<half>(in[6], kernel.packet[6]); + pstore<half>(in[7], kernel.packet[7]); + + EIGEN_ALIGN64 half out[8][16]; + + for (int i = 0; i < 8; ++i) { + for (int j = 0; j < 8; ++j) { + out[i][j] = in[j][2 * i]; + } + for (int j = 0; j < 8; ++j) { + out[i][j + 8] = in[j][2 * i + 1]; + } + } + + kernel.packet[0] = pload<Packet16h>(out[0]); + kernel.packet[1] = pload<Packet16h>(out[1]); + kernel.packet[2] = pload<Packet16h>(out[2]); + kernel.packet[3] = pload<Packet16h>(out[3]); + kernel.packet[4] = pload<Packet16h>(out[4]); + kernel.packet[5] = pload<Packet16h>(out[5]); + kernel.packet[6] = pload<Packet16h>(out[6]); + kernel.packet[7] = pload<Packet16h>(out[7]); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 4>& kernel) { + EIGEN_ALIGN64 half in[4][16]; + pstore<half>(in[0], kernel.packet[0]); + pstore<half>(in[1], kernel.packet[1]); + pstore<half>(in[2], kernel.packet[2]); + pstore<half>(in[3], kernel.packet[3]); + + EIGEN_ALIGN64 half out[4][16]; + + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 4; ++j) { + out[i][j] = in[j][4 * i]; + } + for (int j = 0; j < 4; ++j) { + out[i][j + 4] = in[j][4 * i + 1]; + } + for (int j = 0; j < 4; ++j) { + out[i][j + 8] = in[j][4 * i + 2]; + } + for (int j = 0; j < 4; ++j) { + out[i][j + 12] = in[j][4 * i + 3]; + } + } + + kernel.packet[0] = pload<Packet16h>(out[0]); + kernel.packet[1] = pload<Packet16h>(out[1]); + kernel.packet[2] = pload<Packet16h>(out[2]); + kernel.packet[3] = pload<Packet16h>(out[3]); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8h, 8>& kernel) { + __m128i a = _mm_castph_si128(kernel.packet[0]); + __m128i b = _mm_castph_si128(kernel.packet[1]); + __m128i c = _mm_castph_si128(kernel.packet[2]); + __m128i d = _mm_castph_si128(kernel.packet[3]); + __m128i e = _mm_castph_si128(kernel.packet[4]); + __m128i f = _mm_castph_si128(kernel.packet[5]); + __m128i g = _mm_castph_si128(kernel.packet[6]); + __m128i h = _mm_castph_si128(kernel.packet[7]); + + __m128i a03b03 = _mm_unpacklo_epi16(a, b); + __m128i c03d03 = _mm_unpacklo_epi16(c, d); + __m128i e03f03 = _mm_unpacklo_epi16(e, f); + __m128i g03h03 = _mm_unpacklo_epi16(g, h); + __m128i a47b47 = _mm_unpackhi_epi16(a, b); + __m128i c47d47 = _mm_unpackhi_epi16(c, d); + __m128i e47f47 = _mm_unpackhi_epi16(e, f); + __m128i g47h47 = _mm_unpackhi_epi16(g, h); + + __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03); + __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03); + __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03); + __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03); + __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47); + __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47); + __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47); + __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47); + + __m128i a0b0c0d0e0f0g0h0 = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01); + __m128i a1b1c1d1e1f1g1h1 = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01); + __m128i a2b2c2d2e2f2g2h2 = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23); + __m128i a3b3c3d3e3f3g3h3 = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23); + __m128i a4b4c4d4e4f4g4h4 = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45); + __m128i a5b5c5d5e5f5g5h5 = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45); + __m128i a6b6c6d6e6f6g6h6 = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67); + __m128i a7b7c7d7e7f7g7h7 = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67); + + kernel.packet[0] = _mm_castsi128_ph(a0b0c0d0e0f0g0h0); + kernel.packet[1] = _mm_castsi128_ph(a1b1c1d1e1f1g1h1); + kernel.packet[2] = _mm_castsi128_ph(a2b2c2d2e2f2g2h2); + kernel.packet[3] = _mm_castsi128_ph(a3b3c3d3e3f3g3h3); + kernel.packet[4] = _mm_castsi128_ph(a4b4c4d4e4f4g4h4); + kernel.packet[5] = _mm_castsi128_ph(a5b5c5d5e5f5g5h5); + kernel.packet[6] = _mm_castsi128_ph(a6b6c6d6e6f6g6h6); + kernel.packet[7] = _mm_castsi128_ph(a7b7c7d7e7f7g7h7); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8h, 4>& kernel) { + EIGEN_ALIGN32 Eigen::half in[4][8]; + pstore<Eigen::half>(in[0], kernel.packet[0]); + pstore<Eigen::half>(in[1], kernel.packet[1]); + pstore<Eigen::half>(in[2], kernel.packet[2]); + pstore<Eigen::half>(in[3], kernel.packet[3]); + + EIGEN_ALIGN32 Eigen::half out[4][8]; + + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 4; ++j) { + out[i][j] = in[j][2 * i]; + } + for (int j = 0; j < 4; ++j) { + out[i][j + 4] = in[j][2 * i + 1]; + } + } + + kernel.packet[0] = pload<Packet8h>(out[0]); + kernel.packet[1] = pload<Packet8h>(out[1]); + kernel.packet[2] = pload<Packet8h>(out[2]); + kernel.packet[3] = pload<Packet8h>(out[3]); +} + +// preverse + +template <> +EIGEN_STRONG_INLINE Packet32h preverse(const Packet32h& a) { + return _mm512_permutexvar_ph(_mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31), + a); +} + +template <> +EIGEN_STRONG_INLINE Packet16h preverse(const Packet16h& a) { + __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); + return _mm256_castsi256_ph(_mm256_insertf128_si256( + _mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castph_si256(a), 1), m)), + _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castph_si256(a), 0), m), 1)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h preverse(const Packet8h& a) { + __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); + return _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(a), m)); +} + +// pscatter + +template <> +EIGEN_STRONG_INLINE void pscatter<half, Packet32h>(half* to, const Packet32h& from, Index stride) { + EIGEN_ALIGN64 half aux[32]; + pstore(aux, from); + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 32; i++) { + to[stride * i] = aux[i]; + } +} +template <> +EIGEN_STRONG_INLINE void pscatter<half, Packet16h>(half* to, const Packet16h& from, Index stride) { + EIGEN_ALIGN64 half aux[16]; + pstore(aux, from); + to[stride * 0] = aux[0]; + to[stride * 1] = aux[1]; + to[stride * 2] = aux[2]; + to[stride * 3] = aux[3]; + to[stride * 4] = aux[4]; + to[stride * 5] = aux[5]; + to[stride * 6] = aux[6]; + to[stride * 7] = aux[7]; + to[stride * 8] = aux[8]; + to[stride * 9] = aux[9]; + to[stride * 10] = aux[10]; + to[stride * 11] = aux[11]; + to[stride * 12] = aux[12]; + to[stride * 13] = aux[13]; + to[stride * 14] = aux[14]; + to[stride * 15] = aux[15]; +} + +template <> +EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet8h>(Eigen::half* to, const Packet8h& from, Index stride) { + EIGEN_ALIGN32 Eigen::half aux[8]; + pstore(aux, from); + to[stride * 0] = aux[0]; + to[stride * 1] = aux[1]; + to[stride * 2] = aux[2]; + to[stride * 3] = aux[3]; + to[stride * 4] = aux[4]; + to[stride * 5] = aux[5]; + to[stride * 6] = aux[6]; + to[stride * 7] = aux[7]; +} + +// pgather + +template <> +EIGEN_STRONG_INLINE Packet32h pgather<Eigen::half, Packet32h>(const Eigen::half* from, Index stride) { + return _mm512_set_ph(from[31 * stride].x, from[30 * stride].x, from[29 * stride].x, from[28 * stride].x, + from[27 * stride].x, from[26 * stride].x, from[25 * stride].x, from[24 * stride].x, + from[23 * stride].x, from[22 * stride].x, from[21 * stride].x, from[20 * stride].x, + from[19 * stride].x, from[18 * stride].x, from[17 * stride].x, from[16 * stride].x, + from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x, + from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x, + from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, + from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(const Eigen::half* from, Index stride) { + return _mm256_set_ph(from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x, + from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x, + from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, + from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pgather<Eigen::half, Packet8h>(const Eigen::half* from, Index stride) { + return _mm_set_ph(from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, from[3 * stride].x, + from[2 * stride].x, from[1 * stride].x, from[0 * stride].x); +} + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_PACKET_MATH_FP16_AVX512_H
diff --git a/Eigen/src/Core/arch/AltiVec/PacketMath.h b/Eigen/src/Core/arch/AltiVec/PacketMath.h index d7bd9be..eefe326 100644 --- a/Eigen/src/Core/arch/AltiVec/PacketMath.h +++ b/Eigen/src/Core/arch/AltiVec/PacketMath.h
@@ -185,6 +185,8 @@ HasLog = 1, HasExp = 1, #ifdef EIGEN_VECTORIZE_VSX + HasCmp = 1, + HasPow = 1, HasSqrt = 1, HasCbrt = 1, #if !EIGEN_COMP_CLANG @@ -3175,6 +3177,7 @@ HasATanh = 1, HasATan = 0, HasLog = 0, + HasCmp = 1, HasExp = 1, HasSqrt = 1, HasCbrt = 1,
diff --git a/Eigen/src/Core/arch/AltiVec/TypeCasting.h b/Eigen/src/Core/arch/AltiVec/TypeCasting.h index fdabeb9..439339e 100644 --- a/Eigen/src/Core/arch/AltiVec/TypeCasting.h +++ b/Eigen/src/Core/arch/AltiVec/TypeCasting.h
@@ -129,30 +129,20 @@ } #ifdef EIGEN_VECTORIZE_VSX -// VSX support varies between different compilers and even different -// versions of the same compiler. For gcc version >= 4.9.3, we can use -// vec_cts to efficiently convert Packet2d to Packet2l. Otherwise, use -// a slow version that works with older compilers. -// Update: apparently vec_cts/vec_ctf intrinsics for 64-bit doubles -// are buggy, https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70963 template <> inline Packet2l pcast<Packet2d, Packet2l>(const Packet2d& x) { -#if EIGEN_GNUC_STRICT_AT_LEAST(7, 1, 0) - return vec_cts(x, 0); // TODO: check clang version. -#else - double tmp[2]; - memcpy(tmp, &x, sizeof(tmp)); - Packet2l l = {static_cast<long long>(tmp[0]), static_cast<long long>(tmp[1])}; - return l; -#endif + EIGEN_ALIGN_MAX double dtmp[2]; + pstore(dtmp, x); + EIGEN_ALIGN_MAX long long itmp[2] = {static_cast<long long>(dtmp[0]), static_cast<long long>(dtmp[1])}; + return vec_xl(0, itmp); } template <> inline Packet2d pcast<Packet2l, Packet2d>(const Packet2l& x) { - unsigned long long tmp[2]; - memcpy(tmp, &x, sizeof(tmp)); - Packet2d d = {static_cast<double>(tmp[0]), static_cast<double>(tmp[1])}; - return d; + EIGEN_ALIGN_MAX long long itmp[2]; + vec_xst(x, 0, itmp); + EIGEN_ALIGN_MAX double dtmp[2] = {static_cast<double>(itmp[0]), static_cast<double>(itmp[1])}; + return pload<Packet2d>(dtmp); } #endif
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h b/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h index e9f564b..a46a8ef 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h
@@ -1689,7 +1689,8 @@ } template <typename Packet> -struct psign_impl<Packet, std::enable_if_t<!NumTraits<typename unpacket_traits<Packet>::type>::IsComplex && +struct psign_impl<Packet, std::enable_if_t<!is_scalar<Packet>::value && + !NumTraits<typename unpacket_traits<Packet>::type>::IsComplex && !NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>> { static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { using Scalar = typename unpacket_traits<Packet>::type; @@ -1705,7 +1706,8 @@ }; template <typename Packet> -struct psign_impl<Packet, std::enable_if_t<!NumTraits<typename unpacket_traits<Packet>::type>::IsComplex && +struct psign_impl<Packet, std::enable_if_t<!is_scalar<Packet>::value && + !NumTraits<typename unpacket_traits<Packet>::type>::IsComplex && NumTraits<typename unpacket_traits<Packet>::type>::IsSigned && NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>> { static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { @@ -1724,7 +1726,8 @@ }; template <typename Packet> -struct psign_impl<Packet, std::enable_if_t<!NumTraits<typename unpacket_traits<Packet>::type>::IsComplex && +struct psign_impl<Packet, std::enable_if_t<!is_scalar<Packet>::value && + !NumTraits<typename unpacket_traits<Packet>::type>::IsComplex && !NumTraits<typename unpacket_traits<Packet>::type>::IsSigned && NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>> { static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { @@ -1739,7 +1742,8 @@ // \internal \returns the the sign of a complex number z, defined as z / abs(z). template <typename Packet> -struct psign_impl<Packet, std::enable_if_t<NumTraits<typename unpacket_traits<Packet>::type>::IsComplex && +struct psign_impl<Packet, std::enable_if_t<!is_scalar<Packet>::value && + NumTraits<typename unpacket_traits<Packet>::type>::IsComplex && unpacket_traits<Packet>::vectorizable>> { static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { typedef typename unpacket_traits<Packet>::type Scalar; @@ -2176,7 +2180,8 @@ // Generic implementation of pow(x,y). template <typename Packet> -EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_pow(const Packet& x, const Packet& y) { +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS std::enable_if_t<!is_scalar<Packet>::value, Packet> generic_pow( + const Packet& x, const Packet& y) { typedef typename unpacket_traits<Packet>::type Scalar; const Packet cst_inf = pset1<Packet>(NumTraits<Scalar>::infinity()); @@ -2266,6 +2271,12 @@ return pow; } +template <typename Scalar> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS std::enable_if_t<is_scalar<Scalar>::value, Scalar> generic_pow( + const Scalar& x, const Scalar& y) { + return numext::pow(x, y); +} + namespace unary_pow { template <typename ScalarExponent, bool IsInteger = NumTraits<ScalarExponent>::IsInteger> @@ -2347,35 +2358,36 @@ } template <typename Packet> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet gen_pow(const Packet& x, - const typename unpacket_traits<Packet>::type& exponent) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<!is_scalar<Packet>::value, Packet> gen_pow( + const Packet& x, const typename unpacket_traits<Packet>::type& exponent) { const Packet exponent_packet = pset1<Packet>(exponent); return generic_pow_impl(x, exponent_packet); } +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<is_scalar<Scalar>::value, Scalar> gen_pow( + const Scalar& x, const Scalar& exponent) { + return numext::pow(x, exponent); +} + template <typename Packet, typename ScalarExponent> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_nonint_nonint_errors(const Packet& x, const Packet& powx, const ScalarExponent& exponent) { using Scalar = typename unpacket_traits<Packet>::type; // non-integer base and exponent case - - const Scalar pos_zero = Scalar(0); - const Scalar all_ones = ptrue<Scalar>(Scalar()); - const Scalar pos_one = Scalar(1); - const Scalar pos_inf = NumTraits<Scalar>::infinity(); - const Packet cst_pos_zero = pzero(x); - const Packet cst_pos_one = pset1<Packet>(pos_one); - const Packet cst_pos_inf = pset1<Packet>(pos_inf); + const Packet cst_pos_one = pset1<Packet>(Scalar(1)); + const Packet cst_pos_inf = pset1<Packet>(NumTraits<Scalar>::infinity()); + const Packet cst_true = ptrue<Packet>(x); const bool exponent_is_not_fin = !(numext::isfinite)(exponent); const bool exponent_is_neg = exponent < ScalarExponent(0); const bool exponent_is_pos = exponent > ScalarExponent(0); - const Packet exp_is_not_fin = pset1<Packet>(exponent_is_not_fin ? all_ones : pos_zero); - const Packet exp_is_neg = pset1<Packet>(exponent_is_neg ? all_ones : pos_zero); - const Packet exp_is_pos = pset1<Packet>(exponent_is_pos ? all_ones : pos_zero); + const Packet exp_is_not_fin = exponent_is_not_fin ? cst_true : cst_pos_zero; + const Packet exp_is_neg = exponent_is_neg ? cst_true : cst_pos_zero; + const Packet exp_is_pos = exponent_is_pos ? cst_true : cst_pos_zero; const Packet exp_is_inf = pand(exp_is_not_fin, por(exp_is_neg, exp_is_pos)); const Packet exp_is_nan = pandnot(exp_is_not_fin, por(exp_is_neg, exp_is_pos)); @@ -2411,22 +2423,15 @@ // This routine handles negative exponents. // The return value is either 0, 1, or -1. - - const Scalar pos_zero = Scalar(0); - const Scalar all_ones = ptrue<Scalar>(Scalar()); - const Scalar pos_one = Scalar(1); - - const Packet cst_pos_one = pset1<Packet>(pos_one); - + const Packet cst_pos_one = pset1<Packet>(Scalar(1)); const bool exponent_is_odd = exponent % ScalarExponent(2) != ScalarExponent(0); - - const Packet exp_is_odd = pset1<Packet>(exponent_is_odd ? all_ones : pos_zero); + const Packet exp_is_odd = exponent_is_odd ? ptrue<Packet>(x) : pzero<Packet>(x); const Packet abs_x = pabs(x); const Packet abs_x_is_one = pcmp_eq(abs_x, cst_pos_one); Packet result = pselect(exp_is_odd, x, abs_x); - result = pand(abs_x_is_one, result); + result = pselect(abs_x_is_one, result, pzero<Packet>(x)); return result; }
diff --git a/Eigen/src/Core/arch/GPU/PacketMath.h b/Eigen/src/Core/arch/GPU/PacketMath.h index 6d4230a..328b1b9 100644 --- a/Eigen/src/Core/arch/GPU/PacketMath.h +++ b/Eigen/src/Core/arch/GPU/PacketMath.h
@@ -31,6 +31,15 @@ #define EIGEN_GPU_HAS_FP16_ARITHMETIC 1 #endif +// We need to distinguish ‘clang as the CUDA compiler’ from ‘clang as the host compiler, +// invoked by NVCC’ (e.g. on MacOS). The former needs to see both host and device implementation +// of the functions, while the latter can only deal with one of them. +#if defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC) && EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) +#define EIGEN_HAS_GPU_DEVICE_FUNCTIONS 1 +#else +#define EIGEN_HAS_GPU_DEVICE_FUNCTIONS 0 +#endif + // Make sure this is only available when targeting a GPU: we don't want to // introduce conflicts between these packet_traits definitions and the ones // we'll use on the host side (SSE, AVX, ...) @@ -74,7 +83,10 @@ HasGammaSampleDerAlpha = 1, HasIGammac = 1, HasBetaInc = 1, - HasBlend = 0 + + HasBlend = 0, + HasFloor = 1, + HasCmp = EIGEN_HAS_GPU_DEVICE_FUNCTIONS }; }; @@ -143,10 +155,7 @@ return make_double2(from, from); } -// We need to distinguish ‘clang as the CUDA compiler’ from ‘clang as the host compiler, -// invoked by NVCC’ (e.g. on MacOS). The former needs to see both host and device implementation -// of the functions, while the latter can only deal with one of them. -#if defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC) && EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) +#if EIGEN_HAS_GPU_DEVICE_FUNCTIONS EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float bitwise_and(const float& a, const float& b) { return __int_as_float(__float_as_int(a) & __float_as_int(b)); @@ -259,8 +268,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_le<double2>(const double2& a, const double2& b) { return make_double2(le_mask(a.x, b.x), le_mask(a.y, b.y)); } -#endif // defined(EIGEN_CUDA_ARCH) || defined(EIGEN_HIPCC) || (defined(EIGEN_CUDACC) && EIGEN_COMP_CLANG && - // !EIGEN_COMP_NVCC) +#endif // EIGEN_HAS_GPU_DEVICE_FUNCTIONS template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 plset<float4>(const float& a) {
diff --git a/Eigen/src/Core/arch/HVX/PacketMath.h b/Eigen/src/Core/arch/HVX/PacketMath.h index ccba96e..b9080d9 100644 --- a/Eigen/src/Core/arch/HVX/PacketMath.h +++ b/Eigen/src/Core/arch/HVX/PacketMath.h
@@ -401,7 +401,7 @@ template <HVXPacketSize T> EIGEN_STRONG_INLINE HVXPacket<T> pcmp_le_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) { - HVX_Vector v_true = Q6_Vb_vsplat_R(0xff); + HVX_Vector v_true = Q6_V_vsplat_R(0x3f800000); HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(a.Get(), b.Get()); return HVXPacket<T>::Create(Q6_V_vmux_QVV(pred, Q6_V_vzero(), v_true)); } @@ -420,7 +420,7 @@ template <HVXPacketSize T> EIGEN_STRONG_INLINE HVXPacket<T> pcmp_eq_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) { - HVX_Vector v_true = Q6_Vb_vsplat_R(0xff); + HVX_Vector v_true = Q6_V_vsplat_R(0x3f800000); HVX_VectorPred pred = Q6_Q_vcmp_eq_VwVw(a.Get(), b.Get()); return HVXPacket<T>::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero())); } @@ -439,7 +439,7 @@ template <HVXPacketSize T> EIGEN_STRONG_INLINE HVXPacket<T> pcmp_lt_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) { - HVX_Vector v_true = Q6_Vb_vsplat_R(0xff); + HVX_Vector v_true = Q6_V_vsplat_R(0x3f800000); HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(b.Get(), a.Get()); return HVXPacket<T>::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero())); } @@ -458,7 +458,7 @@ template <HVXPacketSize T> EIGEN_STRONG_INLINE HVXPacket<T> pcmp_lt_or_nan_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) { - HVX_Vector v_true = Q6_Vb_vsplat_R(0xff); + HVX_Vector v_true = Q6_V_vsplat_R(0x3f800000); HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(b.Get(), a.Get()); return HVXPacket<T>::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero())); }
diff --git a/Eigen/src/Core/arch/NEON/Complex.h b/Eigen/src/Core/arch/NEON/Complex.h index 4190d1b..f3f6a1a 100644 --- a/Eigen/src/Core/arch/NEON/Complex.h +++ b/Eigen/src/Core/arch/NEON/Complex.h
@@ -73,30 +73,13 @@ }; template <> -struct unpacket_traits<Packet1cf> { - typedef std::complex<float> type; - typedef Packet1cf half; - typedef Packet2f as_real; - enum { - size = 1, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet1cf> : neon_unpacket_default<Packet1cf, std::complex<float>> { + using as_real = Packet2f; }; template <> -struct unpacket_traits<Packet2cf> { - typedef std::complex<float> type; - typedef Packet1cf half; - typedef Packet4f as_real; - enum { - size = 2, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet2cf> : neon_unpacket_default<Packet2cf, std::complex<float>> { + using half = Packet1cf; + using as_real = Packet4f; }; template <> @@ -297,10 +280,12 @@ template <> EIGEN_STRONG_INLINE Packet1cf pload<Packet1cf>(const std::complex<float>* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet1cf>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return Packet1cf(pload<Packet2f>((const float*)from)); } template <> EIGEN_STRONG_INLINE Packet2cf pload<Packet2cf>(const std::complex<float>* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet2cf>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload<Packet4f>(reinterpret_cast<const float*>(from))); } @@ -324,10 +309,12 @@ template <> EIGEN_STRONG_INLINE void pstore<std::complex<float> >(std::complex<float>* to, const Packet1cf& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet1cf>::alignment); EIGEN_DEBUG_ALIGNED_STORE pstore((float*)to, from.v); } template <> EIGEN_STRONG_INLINE void pstore<std::complex<float> >(std::complex<float>* to, const Packet2cf& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet2cf>::alignment); EIGEN_DEBUG_ALIGNED_STORE pstore(reinterpret_cast<float*>(to), from.v); } @@ -538,21 +525,13 @@ }; template <> -struct unpacket_traits<Packet1cd> { - typedef std::complex<double> type; - typedef Packet1cd half; - typedef Packet2d as_real; - enum { - size = 1, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet1cd> : neon_unpacket_default<Packet1cd, std::complex<double>> { + using as_real = Packet2d; }; template <> EIGEN_STRONG_INLINE Packet1cd pload<Packet1cd>(const std::complex<double>* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet1cd>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload<Packet2d>(reinterpret_cast<const double*>(from))); } @@ -666,6 +645,7 @@ template <> EIGEN_STRONG_INLINE void pstore<std::complex<double> >(std::complex<double>* to, const Packet1cd& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet1cd>::alignment); EIGEN_DEBUG_ALIGNED_STORE pstore(reinterpret_cast<double*>(to), from.v); }
diff --git a/Eigen/src/Core/arch/NEON/PacketMath.h b/Eigen/src/Core/arch/NEON/PacketMath.h index 9364cff..bea50a3 100644 --- a/Eigen/src/Core/arch/NEON/PacketMath.h +++ b/Eigen/src/Core/arch/NEON/PacketMath.h
@@ -205,6 +205,7 @@ HasATanh = 1, HasLog = 1, HasExp = 1, + HasPow = 1, HasSqrt = 1, HasRsqrt = 1, HasCbrt = 1, @@ -437,224 +438,74 @@ }; }; +template <typename Packet, typename Scalar> +struct neon_unpacket_default { + using type = Scalar; + using half = Packet; + static constexpr int size = sizeof(Packet) / sizeof(Scalar); + static constexpr int alignment = sizeof(Packet); + static constexpr bool vectorizable = true; + static constexpr bool masked_load_available = false; + static constexpr bool masked_store_available = false; +}; + template <> -struct unpacket_traits<Packet2f> { - typedef float type; - typedef Packet2f half; - typedef Packet2i integer_packet; - enum { - size = 2, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet2f> : neon_unpacket_default<Packet2f, float> { + using integer_packet = Packet2i; }; template <> -struct unpacket_traits<Packet4f> { - typedef float type; - typedef Packet2f half; - typedef Packet4i integer_packet; - enum { - size = 4, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet4f> : neon_unpacket_default<Packet4f, float> { + using half = Packet2f; + using integer_packet = Packet4i; }; template <> -struct unpacket_traits<Packet4c> { - typedef int8_t type; - typedef Packet4c half; - enum { - size = 4, - alignment = Unaligned, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet4c> : neon_unpacket_default<Packet4c, int8_t> {}; +template <> +struct unpacket_traits<Packet8c> : neon_unpacket_default<Packet8c, int8_t> { + using half = Packet4c; }; template <> -struct unpacket_traits<Packet8c> { - typedef int8_t type; - typedef Packet4c half; - enum { - size = 8, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet16c> : neon_unpacket_default<Packet16c, int8_t> { + using half = Packet8c; }; template <> -struct unpacket_traits<Packet16c> { - typedef int8_t type; - typedef Packet8c half; - enum { - size = 16, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet4uc> : neon_unpacket_default<Packet4uc, uint8_t> {}; +template <> +struct unpacket_traits<Packet8uc> : neon_unpacket_default<Packet8uc, uint8_t> { + using half = Packet4uc; }; template <> -struct unpacket_traits<Packet4uc> { - typedef uint8_t type; - typedef Packet4uc half; - enum { - size = 4, - alignment = Unaligned, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet16uc> : neon_unpacket_default<Packet16uc, uint8_t> { + using half = Packet8uc; }; template <> -struct unpacket_traits<Packet8uc> { - typedef uint8_t type; - typedef Packet4uc half; - enum { - size = 8, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet4s> : neon_unpacket_default<Packet4s, int16_t> {}; +template <> +struct unpacket_traits<Packet8s> : neon_unpacket_default<Packet8s, int16_t> { + using half = Packet4s; }; template <> -struct unpacket_traits<Packet16uc> { - typedef uint8_t type; - typedef Packet8uc half; - enum { - size = 16, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet4us> : neon_unpacket_default<Packet4us, uint16_t> {}; +template <> +struct unpacket_traits<Packet8us> : neon_unpacket_default<Packet8us, uint16_t> { + using half = Packet4us; }; template <> -struct unpacket_traits<Packet4s> { - typedef int16_t type; - typedef Packet4s half; - enum { - size = 4, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet2i> : neon_unpacket_default<Packet2i, int32_t> {}; +template <> +struct unpacket_traits<Packet4i> : neon_unpacket_default<Packet4i, int32_t> { + using half = Packet2i; }; template <> -struct unpacket_traits<Packet8s> { - typedef int16_t type; - typedef Packet4s half; - enum { - size = 8, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet2ui> : neon_unpacket_default<Packet2ui, uint32_t> {}; +template <> +struct unpacket_traits<Packet4ui> : neon_unpacket_default<Packet4ui, uint32_t> { + using half = Packet2ui; }; template <> -struct unpacket_traits<Packet4us> { - typedef uint16_t type; - typedef Packet4us half; - enum { - size = 4, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; +struct unpacket_traits<Packet2l> : neon_unpacket_default<Packet2l, int64_t> {}; template <> -struct unpacket_traits<Packet8us> { - typedef uint16_t type; - typedef Packet4us half; - enum { - size = 8, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; -template <> -struct unpacket_traits<Packet2i> { - typedef int32_t type; - typedef Packet2i half; - enum { - size = 2, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; -template <> -struct unpacket_traits<Packet4i> { - typedef int32_t type; - typedef Packet2i half; - enum { - size = 4, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; -template <> -struct unpacket_traits<Packet2ui> { - typedef uint32_t type; - typedef Packet2ui half; - enum { - size = 2, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; -template <> -struct unpacket_traits<Packet4ui> { - typedef uint32_t type; - typedef Packet2ui half; - enum { - size = 4, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; -template <> -struct unpacket_traits<Packet2l> { - typedef int64_t type; - typedef Packet2l half; - enum { - size = 2, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; -template <> -struct unpacket_traits<Packet2ul> { - typedef uint64_t type; - typedef Packet2ul half; - enum { - size = 2, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; +struct unpacket_traits<Packet2ul> : neon_unpacket_default<Packet2ul, uint64_t> {}; template <> EIGEN_STRONG_INLINE Packet2f pzero(const Packet2f& /*a*/) { @@ -2417,10 +2268,12 @@ template <> EIGEN_STRONG_INLINE Packet2f pload<Packet2f>(const float* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet2f>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1_f32(from); } template <> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet4f>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f32(from); } template <> @@ -2431,10 +2284,12 @@ } template <> EIGEN_STRONG_INLINE Packet8c pload<Packet8c>(const int8_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet8c>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1_s8(from); } template <> EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(const int8_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet16c>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s8(from); } template <> @@ -2445,50 +2300,62 @@ } template <> EIGEN_STRONG_INLINE Packet8uc pload<Packet8uc>(const uint8_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet8uc>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1_u8(from); } template <> EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(const uint8_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet16uc>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u8(from); } template <> EIGEN_STRONG_INLINE Packet4s pload<Packet4s>(const int16_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet4s>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1_s16(from); } template <> EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(const int16_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet8s>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s16(from); } template <> EIGEN_STRONG_INLINE Packet4us pload<Packet4us>(const uint16_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet4us>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1_u16(from); } template <> EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(const uint16_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet8us>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u16(from); } template <> EIGEN_STRONG_INLINE Packet2i pload<Packet2i>(const int32_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet2i>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1_s32(from); } template <> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int32_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet4i>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s32(from); } template <> EIGEN_STRONG_INLINE Packet2ui pload<Packet2ui>(const uint32_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet2ui>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1_u32(from); } template <> EIGEN_STRONG_INLINE Packet4ui pload<Packet4ui>(const uint32_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet4ui>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u32(from); } template <> EIGEN_STRONG_INLINE Packet2l pload<Packet2l>(const int64_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet2l>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s64(from); } template <> EIGEN_STRONG_INLINE Packet2ul pload<Packet2ul>(const uint64_t* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet2ul>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u64(from); } @@ -2713,10 +2580,12 @@ template <> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet2f& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet2f>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1_f32(to, from); } template <> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet4f>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to, from); } template <> @@ -2725,10 +2594,12 @@ } template <> EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet8c& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet8c>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1_s8(to, from); } template <> EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet16c& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet16c>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_s8(to, from); } template <> @@ -2737,50 +2608,62 @@ } template <> EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet8uc& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet8uc>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1_u8(to, from); } template <> EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet16uc& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet16uc>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_u8(to, from); } template <> EIGEN_STRONG_INLINE void pstore<int16_t>(int16_t* to, const Packet4s& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet4s>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1_s16(to, from); } template <> EIGEN_STRONG_INLINE void pstore<int16_t>(int16_t* to, const Packet8s& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet8s>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_s16(to, from); } template <> EIGEN_STRONG_INLINE void pstore<uint16_t>(uint16_t* to, const Packet4us& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet4us>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1_u16(to, from); } template <> EIGEN_STRONG_INLINE void pstore<uint16_t>(uint16_t* to, const Packet8us& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet8us>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_u16(to, from); } template <> EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet2i& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet2i>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1_s32(to, from); } template <> EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet4i& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet4i>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to, from); } template <> EIGEN_STRONG_INLINE void pstore<uint32_t>(uint32_t* to, const Packet2ui& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet2ui>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1_u32(to, from); } template <> EIGEN_STRONG_INLINE void pstore<uint32_t>(uint32_t* to, const Packet4ui& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet4ui>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_u32(to, from); } template <> EIGEN_STRONG_INLINE void pstore<int64_t>(int64_t* to, const Packet2l& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet2l>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_s64(to, from); } template <> EIGEN_STRONG_INLINE void pstore<uint64_t>(uint64_t* to, const Packet2ul& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet2ul>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_u64(to, from); } @@ -4801,17 +4684,7 @@ }; template <> -struct unpacket_traits<Packet4bf> { - typedef bfloat16 type; - typedef Packet4bf half; - enum { - size = 4, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; +struct unpacket_traits<Packet4bf> : neon_unpacket_default<Packet4bf, bfloat16> {}; namespace detail { template <> @@ -4866,6 +4739,7 @@ template <> EIGEN_STRONG_INLINE Packet4bf pload<Packet4bf>(const bfloat16* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet4bf>::alignment); return Packet4bf(pload<Packet4us>(reinterpret_cast<const uint16_t*>(from))); } @@ -4876,6 +4750,7 @@ template <> EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet4bf& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet4bf>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from); } @@ -5186,6 +5061,7 @@ #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG HasExp = 1, HasLog = 1, + HasPow = 1, HasATan = 1, HasATanh = 1, #endif @@ -5201,17 +5077,8 @@ }; template <> -struct unpacket_traits<Packet2d> { - typedef double type; - typedef Packet2d half; - typedef Packet2l integer_packet; - enum { - size = 2, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet2d> : neon_unpacket_default<Packet2d, double> { + using integer_packet = Packet2l; }; template <> @@ -5373,6 +5240,7 @@ template <> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet2d>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f64(from); } @@ -5387,6 +5255,7 @@ } template <> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet2d>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_f64(to, from); } @@ -5579,29 +5448,10 @@ }; template <> -struct unpacket_traits<Packet4hf> { - typedef Eigen::half type; - typedef Packet4hf half; - enum { - size = 4, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; -}; - +struct unpacket_traits<Packet4hf> : neon_unpacket_default<Packet4hf, half> {}; template <> -struct unpacket_traits<Packet8hf> { - typedef Eigen::half type; - typedef Packet4hf half; - enum { - size = 8, - alignment = Aligned16, - vectorizable = true, - masked_load_available = false, - masked_store_available = false - }; +struct unpacket_traits<Packet8hf> : neon_unpacket_default<Packet8hf, half> { + using half = Packet4hf; }; template <> @@ -5934,11 +5784,13 @@ template <> EIGEN_STRONG_INLINE Packet8hf pload<Packet8hf>(const Eigen::half* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet8hf>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f16(reinterpret_cast<const float16_t*>(from)); } template <> EIGEN_STRONG_INLINE Packet4hf pload<Packet4hf>(const Eigen::half* from) { + EIGEN_ASSUME_ALIGNED(from, unpacket_traits<Packet4hf>::alignment); EIGEN_DEBUG_ALIGNED_LOAD return vld1_f16(reinterpret_cast<const float16_t*>(from)); } @@ -6014,11 +5866,13 @@ template <> EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet8hf& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet8hf>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1q_f16(reinterpret_cast<float16_t*>(to), from); } template <> EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet4hf& from) { + EIGEN_ASSUME_ALIGNED(to, unpacket_traits<Packet4hf>::alignment); EIGEN_DEBUG_ALIGNED_STORE vst1_f16(reinterpret_cast<float16_t*>(to), from); }
diff --git a/Eigen/src/Core/arch/SSE/PacketMath.h b/Eigen/src/Core/arch/SSE/PacketMath.h index e8902cf..64ba7ba 100644 --- a/Eigen/src/Core/arch/SSE/PacketMath.h +++ b/Eigen/src/Core/arch/SSE/PacketMath.h
@@ -192,6 +192,7 @@ HasExpm1 = 1, HasNdtri = 1, HasExp = 1, + HasPow = 1, HasBessel = 1, HasSqrt = 1, HasRsqrt = 1, @@ -221,6 +222,7 @@ HasErf = EIGEN_FAST_MATH, HasErfc = EIGEN_FAST_MATH, HasExp = 1, + HasPow = 1, HasSqrt = 1, HasRsqrt = 1, HasCbrt = 1,
diff --git a/Eigen/src/Core/arch/SVE/PacketMath.h b/Eigen/src/Core/arch/SVE/PacketMath.h index 952d756..6115d1d 100644 --- a/Eigen/src/Core/arch/SVE/PacketMath.h +++ b/Eigen/src/Core/arch/SVE/PacketMath.h
@@ -354,10 +354,12 @@ HasDiv = 1, + HasCmp = 1, HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasLog = 1, HasExp = 1, + HasPow = 1, HasSqrt = 1, HasTanh = EIGEN_FAST_MATH, HasErf = EIGEN_FAST_MATH,
diff --git a/Eigen/src/Core/arch/ZVector/PacketMath.h b/Eigen/src/Core/arch/ZVector/PacketMath.h index b456813..4d18af0 100644 --- a/Eigen/src/Core/arch/ZVector/PacketMath.h +++ b/Eigen/src/Core/arch/ZVector/PacketMath.h
@@ -180,6 +180,7 @@ AlignedOnScalar = 1, size = 4, + HasCmp = 1, HasAdd = 1, HasSub = 1, HasMul = 1,
diff --git a/Eigen/src/Core/functors/BinaryFunctors.h b/Eigen/src/Core/functors/BinaryFunctors.h index a93b998..b6ecfb5 100644 --- a/Eigen/src/Core/functors/BinaryFunctors.h +++ b/Eigen/src/Core/functors/BinaryFunctors.h
@@ -362,11 +362,7 @@ struct functor_traits<scalar_pow_op<Scalar, Exponent>> { enum { Cost = 5 * NumTraits<Scalar>::MulCost, - PacketAccess = (!NumTraits<Scalar>::IsComplex && !NumTraits<Scalar>::IsInteger && packet_traits<Scalar>::HasExp && - packet_traits<Scalar>::HasLog && packet_traits<Scalar>::HasRound && packet_traits<Scalar>::HasCmp && - // Temporarily disable packet access for half/bfloat16 until - // accuracy is improved. - !is_same<Scalar, half>::value && !is_same<Scalar, bfloat16>::value) + PacketAccess = (!NumTraits<Scalar>::IsComplex && !NumTraits<Scalar>::IsInteger && packet_traits<Scalar>::HasPow) }; };
diff --git a/Eigen/src/Core/products/SelfadjointMatrixVector.h b/Eigen/src/Core/products/SelfadjointMatrixVector.h index 10f6026..580f6a8 100644 --- a/Eigen/src/Core/products/SelfadjointMatrixVector.h +++ b/Eigen/src/Core/products/SelfadjointMatrixVector.h
@@ -164,6 +164,11 @@ enum { LhsUpLo = LhsMode & (Upper | Lower) }; + // Verify that the Rhs is a vector in the correct orientation. + // Otherwise, we break the assumption that we are multiplying + // MxN * Nx1. + static_assert(Rhs::ColsAtCompileTime == 1, "The RHS must be a column vector."); + template <typename Dest> static EIGEN_DEVICE_FUNC void run(Dest& dest, const Lhs& a_lhs, const Rhs& a_rhs, const Scalar& alpha) { typedef typename Dest::Scalar ResScalar;
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h b/Eigen/src/Core/util/GpuHipCudaDefines.inc similarity index 95% rename from unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h rename to Eigen/src/Core/util/GpuHipCudaDefines.inc index 3073272..4e10500 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h +++ b/Eigen/src/Core/util/GpuHipCudaDefines.inc
@@ -8,7 +8,7 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#if defined(EIGEN_USE_GPU) && !defined(EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H) +#if defined(EIGEN_USE_GPU) && !defined(EIGEN_CORE_GPU_HIP_CUDA_DEFINES_H) #define EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H // Note that we are using EIGEN_USE_HIP here instead of EIGEN_HIPCC...this is by design @@ -98,4 +98,4 @@ #endif // gpu_assert -#endif // EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H +#endif // EIGEN_CORE_GPU_HIP_CUDA_DEFINES_H
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaUndefines.h b/Eigen/src/Core/util/GpuHipCudaUndefines.inc similarity index 86% rename from unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaUndefines.h rename to Eigen/src/Core/util/GpuHipCudaUndefines.inc index 509bcee..342a323 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaUndefines.h +++ b/Eigen/src/Core/util/GpuHipCudaUndefines.inc
@@ -8,7 +8,7 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#if defined(EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H) +#if defined(EIGEN_CORE_GPU_HIP_CUDA_DEFINES_H) #ifndef EIGEN_PERMANENTLY_ENABLE_GPU_HIP_CUDA_DEFINES @@ -40,6 +40,6 @@ #endif // EIGEN_PERMANENTLY_ENABLE_GPU_HIP_CUDA_DEFINES -#undef EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H +#undef EIGEN_CORE_GPU_HIP_CUDA_DEFINES_H -#endif // EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H +#endif // EIGEN_CORE_GPU_HIP_CUDA_DEFINES_H
diff --git a/Eigen/src/Core/util/Macros.h b/Eigen/src/Core/util/Macros.h index aed8a88..b6a64d4 100644 --- a/Eigen/src/Core/util/Macros.h +++ b/Eigen/src/Core/util/Macros.h
@@ -997,8 +997,9 @@ #endif #if !defined(EIGEN_OPTIMIZATION_BARRIER) -#if EIGEN_COMP_GNUC - // According to https://gcc.gnu.org/onlinedocs/gcc/Constraints.html: +// Implement the barrier on GNUC compilers or clang-cl. +#if EIGEN_COMP_GNUC || (defined(__clang__) && defined(_MSC_VER)) +// According to https://gcc.gnu.org/onlinedocs/gcc/Constraints.html: // X: Any operand whatsoever. // r: A register operand is allowed provided that it is in a general // register. @@ -1031,37 +1032,37 @@ // directly for std::complex<T>, Eigen::half, Eigen::bfloat16. For these, // you will need to apply to the underlying POD type. #if EIGEN_ARCH_PPC && EIGEN_COMP_GNUC_STRICT - // This seems to be broken on clang. Packet4f is loaded into a single +// This seems to be broken on clang. Packet4f is loaded into a single // register rather than a vector, zeroing out some entries. Integer // types also generate a compile error. #if EIGEN_OS_MAC - // General, Altivec for Apple (VSX were added in ISA v2.06): +// General, Altivec for Apple (VSX were added in ISA v2.06): #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__("" : "+r,v"(X)); #else - // General, Altivec, VSX otherwise: +// General, Altivec, VSX otherwise: #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__("" : "+r,v,wa"(X)); #endif #elif EIGEN_ARCH_ARM_OR_ARM64 #ifdef __ARM_FP - // General, VFP or NEON. +// General, VFP or NEON. // Clang doesn't like "r", // error: non-trivial scalar-to-vector conversion, possible invalid // constraint for vector typ #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__("" : "+g,w"(X)); #else - // Arm without VFP or NEON. +// Arm without VFP or NEON. // "w" constraint will not compile. #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__("" : "+g"(X)); #endif #elif EIGEN_ARCH_i386_OR_x86_64 - // General, SSE. +// General, SSE. #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__("" : "+g,x"(X)); #else - // Not implemented for other architectures. +// Not implemented for other architectures. #define EIGEN_OPTIMIZATION_BARRIER(X) #endif #else - // Not implemented for other compilers. +// Not implemented for other compilers. #define EIGEN_OPTIMIZATION_BARRIER(X) #endif #endif
diff --git a/Eigen/src/Core/util/Memory.h b/Eigen/src/Core/util/Memory.h index 44056b3..d6c09a3 100644 --- a/Eigen/src/Core/util/Memory.h +++ b/Eigen/src/Core/util/Memory.h
@@ -1339,6 +1339,21 @@ } #endif +/** \internal + * This informs the implementation that PTR is aligned to at least ALIGN_BYTES + */ +#ifndef EIGEN_ASSUME_ALIGNED +#if defined(__cpp_lib_assume_aligned) && (__cpp_lib_assume_aligned >= 201811L) +#define EIGEN_ASSUME_ALIGNED(PTR, ALIGN_BYTES) \ + { PTR = std::assume_aligned<8 * (ALIGN_BYTES)>(PTR); } +#elif EIGEN_HAS_BUILTIN(__builtin_assume_aligned) +#define EIGEN_ASSUME_ALIGNED(PTR, ALIGN_BYTES) \ + { PTR = static_cast<decltype(PTR)>(__builtin_assume_aligned(PTR, (ALIGN_BYTES))); } +#else +#define EIGEN_ASSUME_ALIGNED(PTR, ALIGN_BYTES) /* do nothing */ +#endif +#endif + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h b/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h index 26cd38e..3c65541 100644 --- a/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h +++ b/Eigen/src/SparseCholesky/SimplicialCholesky_impl.h
@@ -274,6 +274,10 @@ } }; +// Symbol is ODR-used, so we need a definition. +template <typename Scalar, typename StorageIndex> +constexpr StorageIndex simpl_chol_helper<Scalar, StorageIndex>::kEmpty; + } // namespace internal template <typename Derived>
diff --git a/Eigen/src/SparseCore/SparseDot.h b/Eigen/src/SparseCore/SparseDot.h index f040915..8aeebc8 100644 --- a/Eigen/src/SparseCore/SparseDot.h +++ b/Eigen/src/SparseCore/SparseDot.h
@@ -36,10 +36,10 @@ Scalar res1(0); Scalar res2(0); for (; i; ++i) { - res1 += numext::conj(i.value()) * other.coeff(i.index()); + res1 = numext::fma(numext::conj(i.value()), other.coeff(i.index()), res1); ++i; if (i) { - res2 += numext::conj(i.value()) * other.coeff(i.index()); + res2 = numext::fma(numext::conj(i.value()), other.coeff(i.index()), res2); } } return res1 + res2; @@ -67,7 +67,7 @@ Scalar res(0); while (i && j) { if (i.index() == j.index()) { - res += numext::conj(i.value()) * j.value(); + res = numext::fma(numext::conj(i.value()), j.value(), res); ++i; ++j; } else if (i.index() < j.index())
diff --git a/Eigen/src/SparseCore/TriangularSolver.h b/Eigen/src/SparseCore/TriangularSolver.h index 7753a24..10e27d7 100644 --- a/Eigen/src/SparseCore/TriangularSolver.h +++ b/Eigen/src/SparseCore/TriangularSolver.h
@@ -41,7 +41,7 @@ lastVal = it.value(); lastIndex = it.index(); if (lastIndex == i) break; - tmp -= lastVal * other.coeff(lastIndex, col); + tmp = numext::fma(-lastVal, other.coeff(lastIndex, col), tmp); } if (Mode & UnitDiag) other.coeffRef(i, col) = tmp; @@ -75,7 +75,7 @@ } else if (it && it.index() == i) ++it; for (; it; ++it) { - tmp -= it.value() * other.coeff(it.index(), col); + tmp = numext::fma(-it.value(), other.coeff(it.index(), col), tmp); } if (Mode & UnitDiag) @@ -107,7 +107,9 @@ tmp /= it.value(); } if (it && it.index() == i) ++it; - for (; it; ++it) other.coeffRef(it.index(), col) -= tmp * it.value(); + for (; it; ++it) { + other.coeffRef(it.index(), col) = numext::fma(-tmp, it.value(), other.coeffRef(it.index(), col)); + } } } } @@ -135,7 +137,9 @@ other.coeffRef(i, col) /= it.value(); } LhsIterator it(lhsEval, i); - for (; it && it.index() < i; ++it) other.coeffRef(it.index(), col) -= tmp * it.value(); + for (; it && it.index() < i; ++it) { + other.coeffRef(it.index(), col) = numext::fma(-tmp, it.value(), other.coeffRef(it.index(), col)); + } } } } @@ -215,9 +219,13 @@ tempVector.restart(); if (IsLower) { if (it.index() == i) ++it; - for (; it; ++it) tempVector.coeffRef(it.index()) -= ci * it.value(); + for (; it; ++it) { + tempVector.coeffRef(it.index()) = numext::fma(-ci, it.value(), tempVector.coeffRef(it.index())); + } } else { - for (; it && it.index() < i; ++it) tempVector.coeffRef(it.index()) -= ci * it.value(); + for (; it && it.index() < i; ++it) { + tempVector.coeffRef(it.index()) = numext::fma(-ci, it.value(), tempVector.coeffRef(it.index())); + } } } }
diff --git a/Eigen/src/SuperLUSupport/SuperLUSupport.h b/Eigen/src/SuperLUSupport/SuperLUSupport.h index 0c10149..b5d29b2 100644 --- a/Eigen/src/SuperLUSupport/SuperLUSupport.h +++ b/Eigen/src/SuperLUSupport/SuperLUSupport.h
@@ -65,6 +65,24 @@ #ifdef EIGEN_SUPERLU_HAS_ILU // similarly for the incomplete factorization using gsisx +#if defined(SUPERLU_MAJOR_VERSION) && (SUPERLU_MAJOR_VERSION >= 5) +#define DECL_GSISX(PREFIX, FLOATTYPE, KEYTYPE) \ + extern "C" { \ + extern void PREFIX##gsisx(superlu_options_t *, SuperMatrix *, int *, int *, int *, char *, FLOATTYPE *, FLOATTYPE *, \ + SuperMatrix *, SuperMatrix *, void *, int, SuperMatrix *, SuperMatrix *, FLOATTYPE *, \ + FLOATTYPE *, GlobalLU_t *, mem_usage_t *, SuperLUStat_t *, int *); \ + } \ + inline float SuperLU_gsisx(superlu_options_t *options, SuperMatrix *A, int *perm_c, int *perm_r, int *etree, \ + char *equed, FLOATTYPE *R, FLOATTYPE *C, SuperMatrix *L, SuperMatrix *U, void *work, \ + int lwork, SuperMatrix *B, SuperMatrix *X, FLOATTYPE *recip_pivot_growth, \ + FLOATTYPE *rcond, SuperLUStat_t *stats, int *info, KEYTYPE) { \ + mem_usage_t mem_usage; \ + GlobalLU_t gLU; \ + PREFIX##gsisx(options, A, perm_c, perm_r, etree, equed, R, C, L, U, work, lwork, B, X, recip_pivot_growth, rcond, \ + &gLU, &mem_usage, stats, info); \ + return mem_usage.for_lu; /* bytes used by the factor storage */ \ + } +#else // version < 5.0 #define DECL_GSISX(PREFIX, FLOATTYPE, KEYTYPE) \ extern "C" { \ extern void PREFIX##gsisx(superlu_options_t *, SuperMatrix *, int *, int *, int *, char *, FLOATTYPE *, FLOATTYPE *, \ @@ -80,6 +98,7 @@ &mem_usage, stats, info); \ return mem_usage.for_lu; /* bytes used by the factor storage */ \ } +#endif DECL_GSISX(s, float, float) DECL_GSISX(c, float, std::complex<float>)
diff --git a/blas/CMakeLists.txt b/blas/CMakeLists.txt index 4ae0603..4f18992 100644 --- a/blas/CMakeLists.txt +++ b/blas/CMakeLists.txt
@@ -2,14 +2,6 @@ project(EigenBlas CXX) if(EIGEN_BUILD_BLAS) -include(CheckLanguage) -check_language(Fortran) -if(CMAKE_Fortran_COMPILER) - enable_language(Fortran) - set(EIGEN_Fortran_COMPILER_WORKS ON) -else() - set(EIGEN_Fortran_COMPILER_WORKS OFF) -endif() add_custom_target(blas) @@ -18,15 +10,9 @@ f2c/lsame.c f2c/dspmv.c f2c/ssbmv.c f2c/chbmv.c f2c/sspmv.c f2c/zhbmv.c f2c/chpmv.c f2c/dsbmv.c f2c/zhpmv.c f2c/dtbmv.c f2c/stbmv.c f2c/ctbmv.c - f2c/ztbmv.c + f2c/ztbmv.c f2c/complexdots.c ) -if (EIGEN_Fortran_COMPILER_WORKS) - set(EigenBlas_SRCS ${EigenBlas_SRCS} fortran/complexdots.f) -else() - set(EigenBlas_SRCS ${EigenBlas_SRCS} f2c/complexdots.c) -endif() - set(EIGEN_BLAS_TARGETS "") add_library(eigen_blas_static ${EigenBlas_SRCS}) @@ -49,8 +35,6 @@ ARCHIVE DESTINATION lib) endforeach() -if(EIGEN_Fortran_COMPILER_WORKS) - if(EIGEN_BUILD_TESTING) if(EIGEN_LEAVE_TEST_IN_ALL_TARGET) add_subdirectory(testing) # can't do EXCLUDE_FROM_ALL here, breaks CTest @@ -60,4 +44,3 @@ endif() endif() -endif()
diff --git a/blas/fortran/complexdots.f b/blas/fortran/complexdots.f deleted file mode 100644 index a7da51d..0000000 --- a/blas/fortran/complexdots.f +++ /dev/null
@@ -1,43 +0,0 @@ - COMPLEX FUNCTION CDOTC(N,CX,INCX,CY,INCY) - INTEGER INCX,INCY,N - COMPLEX CX(*),CY(*) - COMPLEX RES - EXTERNAL CDOTCW - - CALL CDOTCW(N,CX,INCX,CY,INCY,RES) - CDOTC = RES - RETURN - END - - COMPLEX FUNCTION CDOTU(N,CX,INCX,CY,INCY) - INTEGER INCX,INCY,N - COMPLEX CX(*),CY(*) - COMPLEX RES - EXTERNAL CDOTUW - - CALL CDOTUW(N,CX,INCX,CY,INCY,RES) - CDOTU = RES - RETURN - END - - DOUBLE COMPLEX FUNCTION ZDOTC(N,CX,INCX,CY,INCY) - INTEGER INCX,INCY,N - DOUBLE COMPLEX CX(*),CY(*) - DOUBLE COMPLEX RES - EXTERNAL ZDOTCW - - CALL ZDOTCW(N,CX,INCX,CY,INCY,RES) - ZDOTC = RES - RETURN - END - - DOUBLE COMPLEX FUNCTION ZDOTU(N,CX,INCX,CY,INCY) - INTEGER INCX,INCY,N - DOUBLE COMPLEX CX(*),CY(*) - DOUBLE COMPLEX RES - EXTERNAL ZDOTUW - - CALL ZDOTUW(N,CX,INCX,CY,INCY,RES) - ZDOTU = RES - RETURN - END
diff --git a/blas/testing/CMakeLists.txt b/blas/testing/CMakeLists.txt index 52c23ac..f0ee6a4 100644 --- a/blas/testing/CMakeLists.txt +++ b/blas/testing/CMakeLists.txt
@@ -1,3 +1,13 @@ +include(CheckLanguage) +check_language(Fortran) +if(CMAKE_Fortran_COMPILER) + enable_language(Fortran) + set(EIGEN_Fortran_COMPILER_WORKS ON) +else() + set(EIGEN_Fortran_COMPILER_WORKS OFF) +endif() + +if (EIGEN_Fortran_COMPILER_WORKS) macro(ei_add_blas_test testname) @@ -38,3 +48,5 @@ # add_custom_target(level1) # add_dependencies(level1 sblat1) +endif() +
diff --git a/debug/msvc/eigen.natvis b/debug/msvc/eigen.natvis index 22cf346..da89857 100644 --- a/debug/msvc/eigen.natvis +++ b/debug/msvc/eigen.natvis
@@ -1,235 +1,235 @@ -<?xml version="1.0" encoding="utf-8"?> - -<AutoVisualizer xmlns="http://schemas.microsoft.com/vstudio/debugger/natvis/2010"> - - <!-- Fixed x Fixed Matrix --> - <Type Name="Eigen::Matrix<*,*,*,*,*,*>"> - <AlternativeType Name="Eigen::Array<*,-1,-1,*,*,*>"/> - <DisplayString>[{$T2}, {$T3}] (fixed matrix)</DisplayString> - <Expand> - <ArrayItems Condition="Flags%2"> <!-- row major layout --> - <Rank>2</Rank> - <Size>$i==0 ? $T2 : $T3</Size> - <ValuePointer>m_storage.m_data.array</ValuePointer> - </ArrayItems> - <ArrayItems Condition="!(Flags%2)"> <!-- column major layout --> - <Direction>Backward</Direction> - <Rank>2</Rank> - <Size>$i==0 ? $T2 : $T3</Size> - <ValuePointer>m_storage.m_data.array</ValuePointer> - </ArrayItems> - </Expand> - </Type> - - <!-- 2 x 2 Matrix --> - <Type Name="Eigen::Matrix<*,2,2,*,*,*>"> - <AlternativeType Name="Eigen::Array<*,2,2,*,*,*>"/> - <DisplayString>[2, 2] (fixed matrix)</DisplayString> - <Expand> - <Synthetic Name="[row 0]" Condition="Flags%2"> - <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 0]" Condition="!(Flags%2)"> - <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[2]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 1]" Condition="Flags%2"> - <DisplayString>({m_storage.m_data.array[2]}, {m_storage.m_data.array[3]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 1]" Condition="!(Flags%2)"> - <DisplayString>({m_storage.m_data.array[1]}, {m_storage.m_data.array[3]})</DisplayString> - </Synthetic> - </Expand> - </Type> - - <!-- 3 x 3 Matrix --> - <Type Name="Eigen::Matrix<*,3,3,*,*,*>"> - <AlternativeType Name="Eigen::Array<*,3,3,*,*,*>"/> - <DisplayString>[3, 3] (fixed matrix)</DisplayString> - <Expand> - <Synthetic Name="[row 0]" Condition="Flags%2"> - <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]}, {m_storage.m_data.array[2]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 0]" Condition="!(Flags%2)"> - <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[3]}, {m_storage.m_data.array[6]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 1]" Condition="Flags%2"> - <DisplayString>({m_storage.m_data.array[3]}, {m_storage.m_data.array[4]}, {m_storage.m_data.array[5]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 1]" Condition="!(Flags%2)"> - <DisplayString>({m_storage.m_data.array[1]}, {m_storage.m_data.array[4]}, {m_storage.m_data.array[7]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 2]" Condition="Flags%2"> - <DisplayString>({m_storage.m_data.array[6]}, {m_storage.m_data.array[7]}, {m_storage.m_data.array[8]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 2]" Condition="!(Flags%2)"> - <DisplayString>({m_storage.m_data.array[2]}, {m_storage.m_data.array[5]}, {m_storage.m_data.array[8]})</DisplayString> - </Synthetic> - </Expand> - </Type> - - <!-- 4 x 4 Matrix --> - <Type Name="Eigen::Matrix<*,4,4,*,*,*>"> - <AlternativeType Name="Eigen::Array<*,4,4,*,*,*>"/> - <DisplayString>[4, 4] (fixed matrix)</DisplayString> - <Expand> - <Synthetic Name="[row 0]" Condition="Flags%2"> - <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]}, {m_storage.m_data.array[2]}, {m_storage.m_data.array[3]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 0]" Condition="!(Flags%2)"> - <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[4]}, {m_storage.m_data.array[8]}, {m_storage.m_data.array[12]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 1]" Condition="Flags%2"> - <DisplayString>({m_storage.m_data.array[4]}, {m_storage.m_data.array[5]}, {m_storage.m_data.array[6]}, {m_storage.m_data.array[7]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 1]" Condition="!(Flags%2)"> - <DisplayString>({m_storage.m_data.array[1]}, {m_storage.m_data.array[5]}, {m_storage.m_data.array[9]}, {m_storage.m_data.array[13]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 2]" Condition="Flags%2"> - <DisplayString>({m_storage.m_data.array[8]}, {m_storage.m_data.array[9]}, {m_storage.m_data.array[10]}, {m_storage.m_data.array[11]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 2]" Condition="!(Flags%2)"> - <DisplayString>({m_storage.m_data.array[2]}, {m_storage.m_data.array[6]}, {m_storage.m_data.array[10]}, {m_storage.m_data.array[14]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 3]" Condition="Flags%2"> - <DisplayString>({m_storage.m_data.array[12]}, {m_storage.m_data.array[13]}, {m_storage.m_data.array[14]}, {m_storage.m_data.array[15]})</DisplayString> - </Synthetic> - <Synthetic Name="[row 3]" Condition="!(Flags%2)"> - <DisplayString>({m_storage.m_data.array[3]}, {m_storage.m_data.array[7]}, {m_storage.m_data.array[11]}, {m_storage.m_data.array[15]})</DisplayString> - </Synthetic> - </Expand> - </Type> - - <!-- Dynamic x Dynamic Matrix --> - <Type Name="Eigen::Matrix<*,-1,-1,*,*,*>"> - <AlternativeType Name="Eigen::Array<*,-1,-1,*,*,*>"/> - <DisplayString Condition="m_storage.m_data == 0">empty</DisplayString> - <DisplayString Condition="m_storage.m_data != 0">[{m_storage.m_rows}, {m_storage.m_cols}] (dynamic matrix)</DisplayString> - <Expand> - <ArrayItems Condition="Flags%2"> <!-- row major layout --> - <Rank>2</Rank> - <Size>$i==0 ? m_storage.m_rows : m_storage.m_cols</Size> - <ValuePointer>m_storage.m_data</ValuePointer> - </ArrayItems> - <ArrayItems Condition="!(Flags%2)"> <!-- column major layout --> - <Direction>Backward</Direction> - <Rank>2</Rank> - <Size>$i==0 ? m_storage.m_rows : m_storage.m_cols</Size> - <ValuePointer>m_storage.m_data</ValuePointer> - </ArrayItems> - </Expand> - </Type> - - <!-- Fixed x Dynamic Matrix --> - <Type Name="Eigen::Matrix<*,*,-1,*,*,*>"> - <AlternativeType Name="Eigen::Array<*,*,-1,*,*,*>"/> - <DisplayString Condition="m_storage.m_data == 0">empty</DisplayString> - <DisplayString Condition="m_storage.m_data != 0">[{$T2}, {m_storage.m_cols}] (dynamic column matrix)</DisplayString> - <Expand> - <ArrayItems Condition="Flags%2"> <!-- row major layout --> - <Rank>2</Rank> - <Size>$i==0 ? $T2 : m_storage.m_cols</Size> - <ValuePointer>m_storage.m_data</ValuePointer> - </ArrayItems> - <ArrayItems Condition="!(Flags%2)"> <!-- column major layout --> - <Direction>Backward</Direction> - <Rank>2</Rank> - <Size>$i==0 ? $T2 : m_storage.m_cols</Size> - <ValuePointer>m_storage.m_data</ValuePointer> - </ArrayItems> - </Expand> - </Type> - - <!-- Dynamic x Fixed Matrix --> - <Type Name="Eigen::Matrix<*,-1,*,*,*,*>"> - <AlternativeType Name="Eigen::Array<*,-1,*,*,*,*>"/> - <DisplayString Condition="m_storage.m_data == 0">empty</DisplayString> - <DisplayString Condition="m_storage.m_data != 0">[{m_storage.m_rows}, {$T2}] (dynamic row matrix)</DisplayString> - <Expand> - <ArrayItems Condition="Flags%2"> <!-- row major layout --> - <Rank>2</Rank> - <Size>$i==0 ? m_storage.m_rows : $T2</Size> - <ValuePointer>m_storage.m_data</ValuePointer> - </ArrayItems> - <ArrayItems Condition="!(Flags%2)"> <!-- column major layout --> - <Direction>Backward</Direction> - <Rank>2</Rank> - <Size>$i==0 ? m_storage.m_rows : $T2</Size> - <ValuePointer>m_storage.m_data</ValuePointer> - </ArrayItems> - </Expand> - </Type> - - <!-- Dynamic Column Vector --> - <Type Name="Eigen::Matrix<*,1,-1,*,*,*>"> - <AlternativeType Name="Eigen::Array<*,1,-1,*,*,*>"/> - <DisplayString Condition="m_storage.m_data == 0">empty</DisplayString> - <DisplayString Condition="m_storage.m_data != 0">[{m_storage.m_cols}] (dynamic column vector)</DisplayString> - <Expand> - <Item Name="[size]">m_storage.m_cols</Item> - <ArrayItems> - <Size>m_storage.m_cols</Size> - <ValuePointer>m_storage.m_data</ValuePointer> - </ArrayItems> - </Expand> - </Type> - - <!-- Dynamic Row Vector --> - <Type Name="Eigen::Matrix<*,-1,1,*,*,*>"> - <AlternativeType Name="Eigen::Array<*,-1,1,*,*,*>"/> - <DisplayString Condition="m_storage.m_data == 0">empty</DisplayString> - <DisplayString Condition="m_storage.m_data != 0">[{m_storage.m_rows}] (dynamic row vector)</DisplayString> - <Expand> - <Item Name="[size]">m_storage.m_rows</Item> - <ArrayItems> - <Size>m_storage.m_rows</Size> - <ValuePointer>m_storage.m_data</ValuePointer> - </ArrayItems> - </Expand> - </Type> - - <!-- Fixed Vector --> - <Type Name="Eigen::Matrix<*,1,1,*,*,*>"> - <AlternativeType Name="Eigen::Array<*,1,1,*,*,*>"/> - <DisplayString>[1] ({m_storage.m_data.array[0]})</DisplayString> - <Expand> - <Item Name="[x]">m_storage.m_data.array[0]</Item> - </Expand> - </Type> - - <Type Name="Eigen::Matrix<*,2,1,*,*,*>"> - <AlternativeType Name="Eigen::Matrix<*,1,2,*,*,*>"/> - <AlternativeType Name="Eigen::Array<*,2,1,*,*,*>"/> - <AlternativeType Name="Eigen::Array<*,1,2,*,*,*>"/> - <DisplayString>[2] ({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]})</DisplayString> - <Expand> - <Item Name="[x]">m_storage.m_data.array[0]</Item> - <Item Name="[y]">m_storage.m_data.array[1]</Item> - </Expand> - </Type> - - <Type Name="Eigen::Matrix<*,3,1,*,*,*>"> - <AlternativeType Name="Eigen::Matrix<*,1,3,*,*,*>"/> - <AlternativeType Name="Eigen::Array<*,3,1,*,*,*>"/> - <AlternativeType Name="Eigen::Array<*,1,3,*,*,*>"/> - <DisplayString>[3] ({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]}, {m_storage.m_data.array[2]})</DisplayString> - <Expand> - <Item Name="[x]">m_storage.m_data.array[0]</Item> - <Item Name="[y]">m_storage.m_data.array[1]</Item> - <Item Name="[z]">m_storage.m_data.array[2]</Item> - </Expand> - </Type> - - <Type Name="Eigen::Matrix<*,4,1,*,*,*>"> - <AlternativeType Name="Eigen::Matrix<*,1,4,*,*,*>"/> - <AlternativeType Name="Eigen::Array<*,4,1,*,*,*>"/> - <AlternativeType Name="Eigen::Array<*,1,4,*,*,*>"/> - <DisplayString>[4] ({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]}, {m_storage.m_data.array[2]}, {m_storage.m_data.array[3]})</DisplayString> - <Expand> - <Item Name="[x]">m_storage.m_data.array[0]</Item> - <Item Name="[y]">m_storage.m_data.array[1]</Item> - <Item Name="[z]">m_storage.m_data.array[2]</Item> - <Item Name="[w]">m_storage.m_data.array[3]</Item> - </Expand> - </Type> - -</AutoVisualizer> +<?xml version="1.0" encoding="utf-8"?> + +<AutoVisualizer xmlns="http://schemas.microsoft.com/vstudio/debugger/natvis/2010"> + + <!-- Fixed x Fixed Matrix --> + <Type Name="Eigen::Matrix<*,*,*,*,*,*>"> + <AlternativeType Name="Eigen::Array<*,-1,-1,*,*,*>"/> + <DisplayString>[{$T2}, {$T3}] (fixed matrix)</DisplayString> + <Expand> + <ArrayItems Condition="Flags%2"> <!-- row major layout --> + <Rank>2</Rank> + <Size>$i==0 ? $T2 : $T3</Size> + <ValuePointer>m_storage.m_data.array</ValuePointer> + </ArrayItems> + <ArrayItems Condition="!(Flags%2)"> <!-- column major layout --> + <Direction>Backward</Direction> + <Rank>2</Rank> + <Size>$i==0 ? $T2 : $T3</Size> + <ValuePointer>m_storage.m_data.array</ValuePointer> + </ArrayItems> + </Expand> + </Type> + + <!-- 2 x 2 Matrix --> + <Type Name="Eigen::Matrix<*,2,2,*,*,*>"> + <AlternativeType Name="Eigen::Array<*,2,2,*,*,*>"/> + <DisplayString>[2, 2] (fixed matrix)</DisplayString> + <Expand> + <Synthetic Name="[row 0]" Condition="Flags%2"> + <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 0]" Condition="!(Flags%2)"> + <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[2]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 1]" Condition="Flags%2"> + <DisplayString>({m_storage.m_data.array[2]}, {m_storage.m_data.array[3]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 1]" Condition="!(Flags%2)"> + <DisplayString>({m_storage.m_data.array[1]}, {m_storage.m_data.array[3]})</DisplayString> + </Synthetic> + </Expand> + </Type> + + <!-- 3 x 3 Matrix --> + <Type Name="Eigen::Matrix<*,3,3,*,*,*>"> + <AlternativeType Name="Eigen::Array<*,3,3,*,*,*>"/> + <DisplayString>[3, 3] (fixed matrix)</DisplayString> + <Expand> + <Synthetic Name="[row 0]" Condition="Flags%2"> + <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]}, {m_storage.m_data.array[2]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 0]" Condition="!(Flags%2)"> + <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[3]}, {m_storage.m_data.array[6]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 1]" Condition="Flags%2"> + <DisplayString>({m_storage.m_data.array[3]}, {m_storage.m_data.array[4]}, {m_storage.m_data.array[5]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 1]" Condition="!(Flags%2)"> + <DisplayString>({m_storage.m_data.array[1]}, {m_storage.m_data.array[4]}, {m_storage.m_data.array[7]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 2]" Condition="Flags%2"> + <DisplayString>({m_storage.m_data.array[6]}, {m_storage.m_data.array[7]}, {m_storage.m_data.array[8]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 2]" Condition="!(Flags%2)"> + <DisplayString>({m_storage.m_data.array[2]}, {m_storage.m_data.array[5]}, {m_storage.m_data.array[8]})</DisplayString> + </Synthetic> + </Expand> + </Type> + + <!-- 4 x 4 Matrix --> + <Type Name="Eigen::Matrix<*,4,4,*,*,*>"> + <AlternativeType Name="Eigen::Array<*,4,4,*,*,*>"/> + <DisplayString>[4, 4] (fixed matrix)</DisplayString> + <Expand> + <Synthetic Name="[row 0]" Condition="Flags%2"> + <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]}, {m_storage.m_data.array[2]}, {m_storage.m_data.array[3]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 0]" Condition="!(Flags%2)"> + <DisplayString>({m_storage.m_data.array[0]}, {m_storage.m_data.array[4]}, {m_storage.m_data.array[8]}, {m_storage.m_data.array[12]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 1]" Condition="Flags%2"> + <DisplayString>({m_storage.m_data.array[4]}, {m_storage.m_data.array[5]}, {m_storage.m_data.array[6]}, {m_storage.m_data.array[7]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 1]" Condition="!(Flags%2)"> + <DisplayString>({m_storage.m_data.array[1]}, {m_storage.m_data.array[5]}, {m_storage.m_data.array[9]}, {m_storage.m_data.array[13]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 2]" Condition="Flags%2"> + <DisplayString>({m_storage.m_data.array[8]}, {m_storage.m_data.array[9]}, {m_storage.m_data.array[10]}, {m_storage.m_data.array[11]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 2]" Condition="!(Flags%2)"> + <DisplayString>({m_storage.m_data.array[2]}, {m_storage.m_data.array[6]}, {m_storage.m_data.array[10]}, {m_storage.m_data.array[14]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 3]" Condition="Flags%2"> + <DisplayString>({m_storage.m_data.array[12]}, {m_storage.m_data.array[13]}, {m_storage.m_data.array[14]}, {m_storage.m_data.array[15]})</DisplayString> + </Synthetic> + <Synthetic Name="[row 3]" Condition="!(Flags%2)"> + <DisplayString>({m_storage.m_data.array[3]}, {m_storage.m_data.array[7]}, {m_storage.m_data.array[11]}, {m_storage.m_data.array[15]})</DisplayString> + </Synthetic> + </Expand> + </Type> + + <!-- Dynamic x Dynamic Matrix --> + <Type Name="Eigen::Matrix<*,-1,-1,*,*,*>"> + <AlternativeType Name="Eigen::Array<*,-1,-1,*,*,*>"/> + <DisplayString Condition="m_storage.m_data == 0">empty</DisplayString> + <DisplayString Condition="m_storage.m_data != 0">[{m_storage.m_rows}, {m_storage.m_cols}] (dynamic matrix)</DisplayString> + <Expand> + <ArrayItems Condition="Flags%2"> <!-- row major layout --> + <Rank>2</Rank> + <Size>$i==0 ? m_storage.m_rows : m_storage.m_cols</Size> + <ValuePointer>m_storage.m_data</ValuePointer> + </ArrayItems> + <ArrayItems Condition="!(Flags%2)"> <!-- column major layout --> + <Direction>Backward</Direction> + <Rank>2</Rank> + <Size>$i==0 ? m_storage.m_rows : m_storage.m_cols</Size> + <ValuePointer>m_storage.m_data</ValuePointer> + </ArrayItems> + </Expand> + </Type> + + <!-- Fixed x Dynamic Matrix --> + <Type Name="Eigen::Matrix<*,*,-1,*,*,*>"> + <AlternativeType Name="Eigen::Array<*,*,-1,*,*,*>"/> + <DisplayString Condition="m_storage.m_data == 0">empty</DisplayString> + <DisplayString Condition="m_storage.m_data != 0">[{$T2}, {m_storage.m_cols}] (dynamic column matrix)</DisplayString> + <Expand> + <ArrayItems Condition="Flags%2"> <!-- row major layout --> + <Rank>2</Rank> + <Size>$i==0 ? $T2 : m_storage.m_cols</Size> + <ValuePointer>m_storage.m_data</ValuePointer> + </ArrayItems> + <ArrayItems Condition="!(Flags%2)"> <!-- column major layout --> + <Direction>Backward</Direction> + <Rank>2</Rank> + <Size>$i==0 ? $T2 : m_storage.m_cols</Size> + <ValuePointer>m_storage.m_data</ValuePointer> + </ArrayItems> + </Expand> + </Type> + + <!-- Dynamic x Fixed Matrix --> + <Type Name="Eigen::Matrix<*,-1,*,*,*,*>"> + <AlternativeType Name="Eigen::Array<*,-1,*,*,*,*>"/> + <DisplayString Condition="m_storage.m_data == 0">empty</DisplayString> + <DisplayString Condition="m_storage.m_data != 0">[{m_storage.m_rows}, {$T2}] (dynamic row matrix)</DisplayString> + <Expand> + <ArrayItems Condition="Flags%2"> <!-- row major layout --> + <Rank>2</Rank> + <Size>$i==0 ? m_storage.m_rows : $T2</Size> + <ValuePointer>m_storage.m_data</ValuePointer> + </ArrayItems> + <ArrayItems Condition="!(Flags%2)"> <!-- column major layout --> + <Direction>Backward</Direction> + <Rank>2</Rank> + <Size>$i==0 ? m_storage.m_rows : $T2</Size> + <ValuePointer>m_storage.m_data</ValuePointer> + </ArrayItems> + </Expand> + </Type> + + <!-- Dynamic Column Vector --> + <Type Name="Eigen::Matrix<*,1,-1,*,*,*>"> + <AlternativeType Name="Eigen::Array<*,1,-1,*,*,*>"/> + <DisplayString Condition="m_storage.m_data == 0">empty</DisplayString> + <DisplayString Condition="m_storage.m_data != 0">[{m_storage.m_cols}] (dynamic column vector)</DisplayString> + <Expand> + <Item Name="[size]">m_storage.m_cols</Item> + <ArrayItems> + <Size>m_storage.m_cols</Size> + <ValuePointer>m_storage.m_data</ValuePointer> + </ArrayItems> + </Expand> + </Type> + + <!-- Dynamic Row Vector --> + <Type Name="Eigen::Matrix<*,-1,1,*,*,*>"> + <AlternativeType Name="Eigen::Array<*,-1,1,*,*,*>"/> + <DisplayString Condition="m_storage.m_data == 0">empty</DisplayString> + <DisplayString Condition="m_storage.m_data != 0">[{m_storage.m_rows}] (dynamic row vector)</DisplayString> + <Expand> + <Item Name="[size]">m_storage.m_rows</Item> + <ArrayItems> + <Size>m_storage.m_rows</Size> + <ValuePointer>m_storage.m_data</ValuePointer> + </ArrayItems> + </Expand> + </Type> + + <!-- Fixed Vector --> + <Type Name="Eigen::Matrix<*,1,1,*,*,*>"> + <AlternativeType Name="Eigen::Array<*,1,1,*,*,*>"/> + <DisplayString>[1] ({m_storage.m_data.array[0]})</DisplayString> + <Expand> + <Item Name="[x]">m_storage.m_data.array[0]</Item> + </Expand> + </Type> + + <Type Name="Eigen::Matrix<*,2,1,*,*,*>"> + <AlternativeType Name="Eigen::Matrix<*,1,2,*,*,*>"/> + <AlternativeType Name="Eigen::Array<*,2,1,*,*,*>"/> + <AlternativeType Name="Eigen::Array<*,1,2,*,*,*>"/> + <DisplayString>[2] ({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]})</DisplayString> + <Expand> + <Item Name="[x]">m_storage.m_data.array[0]</Item> + <Item Name="[y]">m_storage.m_data.array[1]</Item> + </Expand> + </Type> + + <Type Name="Eigen::Matrix<*,3,1,*,*,*>"> + <AlternativeType Name="Eigen::Matrix<*,1,3,*,*,*>"/> + <AlternativeType Name="Eigen::Array<*,3,1,*,*,*>"/> + <AlternativeType Name="Eigen::Array<*,1,3,*,*,*>"/> + <DisplayString>[3] ({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]}, {m_storage.m_data.array[2]})</DisplayString> + <Expand> + <Item Name="[x]">m_storage.m_data.array[0]</Item> + <Item Name="[y]">m_storage.m_data.array[1]</Item> + <Item Name="[z]">m_storage.m_data.array[2]</Item> + </Expand> + </Type> + + <Type Name="Eigen::Matrix<*,4,1,*,*,*>"> + <AlternativeType Name="Eigen::Matrix<*,1,4,*,*,*>"/> + <AlternativeType Name="Eigen::Array<*,4,1,*,*,*>"/> + <AlternativeType Name="Eigen::Array<*,1,4,*,*,*>"/> + <DisplayString>[4] ({m_storage.m_data.array[0]}, {m_storage.m_data.array[1]}, {m_storage.m_data.array[2]}, {m_storage.m_data.array[3]})</DisplayString> + <Expand> + <Item Name="[x]">m_storage.m_data.array[0]</Item> + <Item Name="[y]">m_storage.m_data.array[1]</Item> + <Item Name="[z]">m_storage.m_data.array[2]</Item> + <Item Name="[w]">m_storage.m_data.array[3]</Item> + </Expand> + </Type> + +</AutoVisualizer>
diff --git a/debug/msvc/eigen_autoexp_part.dat b/debug/msvc/eigen_autoexp_part.dat index 273c10d..35ef580 100644 --- a/debug/msvc/eigen_autoexp_part.dat +++ b/debug/msvc/eigen_autoexp_part.dat
@@ -1,295 +1,295 @@ -; *************************************************************** -; * Eigen Visualizer -; * -; * Author: Hauke Heibel <hauke.heibel@gmail.com> -; * -; * Support the enhanced debugging of the following Eigen -; * types (*: any, +:fixed dimension) : -; * -; * - Eigen::Matrix<*,4,1,*,*,*> and Eigen::Matrix<*,1,4,*,*,*> -; * - Eigen::Matrix<*,3,1,*,*,*> and Eigen::Matrix<*,1,3,*,*,*> -; * - Eigen::Matrix<*,2,1,*,*,*> and Eigen::Matrix<*,1,2,*,*,*> -; * - Eigen::Matrix<*,-1,-1,*,*,*> -; * - Eigen::Matrix<*,+,-1,*,*,*> -; * - Eigen::Matrix<*,-1,+,*,*,*> -; * - Eigen::Matrix<*,+,+,*,*,*> -; * -; * Matrices are displayed properly independently of the memory -; * alignment (RowMajor vs. ColMajor). -; * -; * This file is distributed WITHOUT ANY WARRANTY. Please ensure -; * that your original autoexp.dat file is copied to a safe -; * place before proceeding with its modification. -; *************************************************************** - -[Visualizer] - -; Fixed size 4-vectors -Eigen::Matrix<*,4,1,*,*,*>|Eigen::Matrix<*,1,4,*,*,*>{ - children - ( - #( - [internals]: [$c,!], - x : ($c.m_storage.m_data.array)[0], - y : ($c.m_storage.m_data.array)[1], - z : ($c.m_storage.m_data.array)[2], - w : ($c.m_storage.m_data.array)[3] - ) - ) - - preview - ( - #( - "[", - 4, - "](", - #array(expr: $e.m_storage.m_data.array[$i], size: 4), - ")" - ) - ) -} - -; Fixed size 3-vectors -Eigen::Matrix<*,3,1,*,*,*>|Eigen::Matrix<*,1,3,*,*,*>{ - children - ( - #( - [internals]: [$c,!], - x : ($c.m_storage.m_data.array)[0], - y : ($c.m_storage.m_data.array)[1], - z : ($c.m_storage.m_data.array)[2] - ) - ) - - preview - ( - #( - "[", - 3, - "](", - #array(expr: $e.m_storage.m_data.array[$i], size: 3), - ")" - ) - ) -} - -; Fixed size 2-vectors -Eigen::Matrix<*,2,1,*,*,*>|Eigen::Matrix<*,1,2,*,*,*>{ - children - ( - #( - [internals]: [$c,!], - x : ($c.m_storage.m_data.array)[0], - y : ($c.m_storage.m_data.array)[1] - ) - ) - - preview - ( - #( - "[", - 2, - "](", - #array(expr: $e.m_storage.m_data.array[$i], size: 2), - ")" - ) - ) -} - -; Fixed size 1-vectors -Eigen::Matrix<*,1,1,*,*,*>|Eigen::Matrix<*,1,1,*,*,*>{ - children - ( - #( - [internals]: [$c,!], - x : ($c.m_storage.m_data.array)[0] - ) - ) - - preview - ( - #( - "[", - 1, - "](", - #array(expr: $e.m_storage.m_data.array[$i], size: 1), - ")" - ) - ) -} - -; Dynamic matrices (ColMajor and RowMajor support) -Eigen::Matrix<*,-1,-1,*,*,*>{ - children - ( - #( - [internals]: [$c,!], - rows: $c.m_storage.m_rows, - cols: $c.m_storage.m_cols, - ; Check for RowMajorBit - #if ($c.Flags & 0x1) ( - #array( - rank: 2, - base: 0, - expr: ($c.m_storage.m_data)[($i % $c.m_storage.m_rows)*$c.m_storage.m_cols + (($i- $i % $c.m_storage.m_rows)/$c.m_storage.m_rows)], - size: ($r==1)*$c.m_storage.m_rows+($r==0)*$c.m_storage.m_cols - ) - ) #else ( - #array( - rank: 2, - base: 0, - expr: ($c.m_storage.m_data)[$i], - size: ($r==1)*$c.m_storage.m_rows+($r==0)*$c.m_storage.m_cols - ) - ) - ) - ) - - preview - ( - #( - "[", - $c.m_storage.m_rows, - ",", - $c.m_storage.m_cols, - "](", - #array( - expr : [($c.m_storage.m_data)[$i],g], - size : $c.m_storage.m_rows*$c.m_storage.m_cols - ), - ")" - ) - ) -} - -; Fixed rows, dynamic columns matrix (ColMajor and RowMajor support) -Eigen::Matrix<*,*,-1,*,*,*>{ - children - ( - #( - [internals]: [$c,!], - rows: $c.RowsAtCompileTime, - cols: $c.m_storage.m_cols, - ; Check for RowMajorBit - #if ($c.Flags & 0x1) ( - #array( - rank: 2, - base: 0, - expr: ($c.m_storage.m_data)[($i % $c.RowsAtCompileTime)*$c.m_storage.m_cols + (($i- $i % $c.RowsAtCompileTime)/$c.RowsAtCompileTime)], - size: ($r==1)*$c.RowsAtCompileTime+($r==0)*$c.m_storage.m_cols - ) - ) #else ( - #array( - rank: 2, - base: 0, - expr: ($c.m_storage.m_data)[$i], - size: ($r==1)*$c.RowsAtCompileTime+($r==0)*$c.m_storage.m_cols - ) - ) - ) - ) - - preview - ( - #( - "[", - $c.RowsAtCompileTime, - ",", - $c.m_storage.m_cols, - "](", - #array( - expr : [($c.m_storage.m_data)[$i],g], - size : $c.RowsAtCompileTime*$c.m_storage.m_cols - ), - ")" - ) - ) -} - -; Dynamic rows, fixed columns matrix (ColMajor and RowMajor support) -Eigen::Matrix<*,-1,*,*,*,*>{ - children - ( - #( - [internals]: [$c,!], - rows: $c.m_storage.m_rows, - cols: $c.ColsAtCompileTime, - ; Check for RowMajorBit - #if ($c.Flags & 0x1) ( - #array( - rank: 2, - base: 0, - expr: ($c.m_storage.m_data)[($i % $c.m_storage.m_rows)*$c.ColsAtCompileTime + (($i- $i % $c.m_storage.m_rows)/$c.m_storage.m_rows)], - size: ($r==1)*$c.m_storage.m_rows+($r==0)*$c.ColsAtCompileTime - ) - ) #else ( - #array( - rank: 2, - base: 0, - expr: ($c.m_storage.m_data)[$i], - size: ($r==1)*$c.m_storage.m_rows+($r==0)*$c.ColsAtCompileTime - ) - ) - ) - ) - - preview - ( - #( - "[", - $c.m_storage.m_rows, - ",", - $c.ColsAtCompileTime, - "](", - #array( - expr : [($c.m_storage.m_data)[$i],g], - size : $c.m_storage.m_rows*$c.ColsAtCompileTime - ), - ")" - ) - ) -} - -; Fixed size matrix (ColMajor and RowMajor support) -Eigen::Matrix<*,*,*,*,*,*>{ - children - ( - #( - [internals]: [$c,!], - rows: $c.RowsAtCompileTime, - cols: $c.ColsAtCompileTime, - ; Check for RowMajorBit - #if ($c.Flags & 0x1) ( - #array( - rank: 2, - base: 0, - expr: ($c.m_storage.m_data.array)[($i % $c.RowsAtCompileTime)*$c.ColsAtCompileTime + (($i- $i % $c.RowsAtCompileTime)/$c.RowsAtCompileTime)], - size: ($r==1)*$c.RowsAtCompileTime+($r==0)*$c.ColsAtCompileTime - ) - ) #else ( - #array( - rank: 2, - base: 0, - expr: ($c.m_storage.m_data.array)[$i], - size: ($r==1)*$c.RowsAtCompileTime+($r==0)*$c.ColsAtCompileTime - ) - ) - ) - ) - - preview - ( - #( - "[", - $c.RowsAtCompileTime, - ",", - $c.ColsAtCompileTime, - "](", - #array( - expr : [($c.m_storage.m_data.array)[$i],g], - size : $c.RowsAtCompileTime*$c.ColsAtCompileTime - ), - ")" - ) - ) -} +; *************************************************************** +; * Eigen Visualizer +; * +; * Author: Hauke Heibel <hauke.heibel@gmail.com> +; * +; * Support the enhanced debugging of the following Eigen +; * types (*: any, +:fixed dimension) : +; * +; * - Eigen::Matrix<*,4,1,*,*,*> and Eigen::Matrix<*,1,4,*,*,*> +; * - Eigen::Matrix<*,3,1,*,*,*> and Eigen::Matrix<*,1,3,*,*,*> +; * - Eigen::Matrix<*,2,1,*,*,*> and Eigen::Matrix<*,1,2,*,*,*> +; * - Eigen::Matrix<*,-1,-1,*,*,*> +; * - Eigen::Matrix<*,+,-1,*,*,*> +; * - Eigen::Matrix<*,-1,+,*,*,*> +; * - Eigen::Matrix<*,+,+,*,*,*> +; * +; * Matrices are displayed properly independently of the memory +; * alignment (RowMajor vs. ColMajor). +; * +; * This file is distributed WITHOUT ANY WARRANTY. Please ensure +; * that your original autoexp.dat file is copied to a safe +; * place before proceeding with its modification. +; *************************************************************** + +[Visualizer] + +; Fixed size 4-vectors +Eigen::Matrix<*,4,1,*,*,*>|Eigen::Matrix<*,1,4,*,*,*>{ + children + ( + #( + [internals]: [$c,!], + x : ($c.m_storage.m_data.array)[0], + y : ($c.m_storage.m_data.array)[1], + z : ($c.m_storage.m_data.array)[2], + w : ($c.m_storage.m_data.array)[3] + ) + ) + + preview + ( + #( + "[", + 4, + "](", + #array(expr: $e.m_storage.m_data.array[$i], size: 4), + ")" + ) + ) +} + +; Fixed size 3-vectors +Eigen::Matrix<*,3,1,*,*,*>|Eigen::Matrix<*,1,3,*,*,*>{ + children + ( + #( + [internals]: [$c,!], + x : ($c.m_storage.m_data.array)[0], + y : ($c.m_storage.m_data.array)[1], + z : ($c.m_storage.m_data.array)[2] + ) + ) + + preview + ( + #( + "[", + 3, + "](", + #array(expr: $e.m_storage.m_data.array[$i], size: 3), + ")" + ) + ) +} + +; Fixed size 2-vectors +Eigen::Matrix<*,2,1,*,*,*>|Eigen::Matrix<*,1,2,*,*,*>{ + children + ( + #( + [internals]: [$c,!], + x : ($c.m_storage.m_data.array)[0], + y : ($c.m_storage.m_data.array)[1] + ) + ) + + preview + ( + #( + "[", + 2, + "](", + #array(expr: $e.m_storage.m_data.array[$i], size: 2), + ")" + ) + ) +} + +; Fixed size 1-vectors +Eigen::Matrix<*,1,1,*,*,*>|Eigen::Matrix<*,1,1,*,*,*>{ + children + ( + #( + [internals]: [$c,!], + x : ($c.m_storage.m_data.array)[0] + ) + ) + + preview + ( + #( + "[", + 1, + "](", + #array(expr: $e.m_storage.m_data.array[$i], size: 1), + ")" + ) + ) +} + +; Dynamic matrices (ColMajor and RowMajor support) +Eigen::Matrix<*,-1,-1,*,*,*>{ + children + ( + #( + [internals]: [$c,!], + rows: $c.m_storage.m_rows, + cols: $c.m_storage.m_cols, + ; Check for RowMajorBit + #if ($c.Flags & 0x1) ( + #array( + rank: 2, + base: 0, + expr: ($c.m_storage.m_data)[($i % $c.m_storage.m_rows)*$c.m_storage.m_cols + (($i- $i % $c.m_storage.m_rows)/$c.m_storage.m_rows)], + size: ($r==1)*$c.m_storage.m_rows+($r==0)*$c.m_storage.m_cols + ) + ) #else ( + #array( + rank: 2, + base: 0, + expr: ($c.m_storage.m_data)[$i], + size: ($r==1)*$c.m_storage.m_rows+($r==0)*$c.m_storage.m_cols + ) + ) + ) + ) + + preview + ( + #( + "[", + $c.m_storage.m_rows, + ",", + $c.m_storage.m_cols, + "](", + #array( + expr : [($c.m_storage.m_data)[$i],g], + size : $c.m_storage.m_rows*$c.m_storage.m_cols + ), + ")" + ) + ) +} + +; Fixed rows, dynamic columns matrix (ColMajor and RowMajor support) +Eigen::Matrix<*,*,-1,*,*,*>{ + children + ( + #( + [internals]: [$c,!], + rows: $c.RowsAtCompileTime, + cols: $c.m_storage.m_cols, + ; Check for RowMajorBit + #if ($c.Flags & 0x1) ( + #array( + rank: 2, + base: 0, + expr: ($c.m_storage.m_data)[($i % $c.RowsAtCompileTime)*$c.m_storage.m_cols + (($i- $i % $c.RowsAtCompileTime)/$c.RowsAtCompileTime)], + size: ($r==1)*$c.RowsAtCompileTime+($r==0)*$c.m_storage.m_cols + ) + ) #else ( + #array( + rank: 2, + base: 0, + expr: ($c.m_storage.m_data)[$i], + size: ($r==1)*$c.RowsAtCompileTime+($r==0)*$c.m_storage.m_cols + ) + ) + ) + ) + + preview + ( + #( + "[", + $c.RowsAtCompileTime, + ",", + $c.m_storage.m_cols, + "](", + #array( + expr : [($c.m_storage.m_data)[$i],g], + size : $c.RowsAtCompileTime*$c.m_storage.m_cols + ), + ")" + ) + ) +} + +; Dynamic rows, fixed columns matrix (ColMajor and RowMajor support) +Eigen::Matrix<*,-1,*,*,*,*>{ + children + ( + #( + [internals]: [$c,!], + rows: $c.m_storage.m_rows, + cols: $c.ColsAtCompileTime, + ; Check for RowMajorBit + #if ($c.Flags & 0x1) ( + #array( + rank: 2, + base: 0, + expr: ($c.m_storage.m_data)[($i % $c.m_storage.m_rows)*$c.ColsAtCompileTime + (($i- $i % $c.m_storage.m_rows)/$c.m_storage.m_rows)], + size: ($r==1)*$c.m_storage.m_rows+($r==0)*$c.ColsAtCompileTime + ) + ) #else ( + #array( + rank: 2, + base: 0, + expr: ($c.m_storage.m_data)[$i], + size: ($r==1)*$c.m_storage.m_rows+($r==0)*$c.ColsAtCompileTime + ) + ) + ) + ) + + preview + ( + #( + "[", + $c.m_storage.m_rows, + ",", + $c.ColsAtCompileTime, + "](", + #array( + expr : [($c.m_storage.m_data)[$i],g], + size : $c.m_storage.m_rows*$c.ColsAtCompileTime + ), + ")" + ) + ) +} + +; Fixed size matrix (ColMajor and RowMajor support) +Eigen::Matrix<*,*,*,*,*,*>{ + children + ( + #( + [internals]: [$c,!], + rows: $c.RowsAtCompileTime, + cols: $c.ColsAtCompileTime, + ; Check for RowMajorBit + #if ($c.Flags & 0x1) ( + #array( + rank: 2, + base: 0, + expr: ($c.m_storage.m_data.array)[($i % $c.RowsAtCompileTime)*$c.ColsAtCompileTime + (($i- $i % $c.RowsAtCompileTime)/$c.RowsAtCompileTime)], + size: ($r==1)*$c.RowsAtCompileTime+($r==0)*$c.ColsAtCompileTime + ) + ) #else ( + #array( + rank: 2, + base: 0, + expr: ($c.m_storage.m_data.array)[$i], + size: ($r==1)*$c.RowsAtCompileTime+($r==0)*$c.ColsAtCompileTime + ) + ) + ) + ) + + preview + ( + #( + "[", + $c.RowsAtCompileTime, + ",", + $c.ColsAtCompileTime, + "](", + #array( + expr : [($c.m_storage.m_data.array)[$i],g], + size : $c.RowsAtCompileTime*$c.ColsAtCompileTime + ), + ")" + ) + ) +}
diff --git a/doc/QuickReference.dox b/doc/QuickReference.dox index c61d47a..1fb000f 100644 --- a/doc/QuickReference.dox +++ b/doc/QuickReference.dox
@@ -449,7 +449,7 @@ </td></tr> </table> -Some coefficient-wise operators are readily available for for matrices and vectors through the following cwise* methods: +Some coefficient-wise operators are readily available for matrices and vectors through the following cwise* methods: <table class="manual"> <tr><th>Matrix API \matrixworld</th><th>Via Array conversions</th></tr> <tr><td>\code
diff --git a/test/AnnoyingScalar.h b/test/AnnoyingScalar.h index 00a20c7..9e320ea 100644 --- a/test/AnnoyingScalar.h +++ b/test/AnnoyingScalar.h
@@ -16,7 +16,7 @@ #pragma GCC diagnostic ignored "-Wshadow" #endif -#ifndef EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW +#if defined(EIGEN_EXCEPTIONS) && !defined(EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW) struct my_exception { my_exception() {} ~my_exception() {} @@ -76,7 +76,7 @@ } AnnoyingScalar operator+(const AnnoyingScalar& other) const { -#ifndef EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW +#if defined(EIGEN_EXCEPTIONS) && !defined(EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW) countdown--; if (countdown <= 0 && !dont_throw) throw my_exception(); #endif
diff --git a/test/array_cwise.cpp b/test/array_cwise.cpp index cf0e6e4..6ff8d67 100644 --- a/test/array_cwise.cpp +++ b/test/array_cwise.cpp
@@ -1340,7 +1340,7 @@ CALL_SUBTEST_3(array_generic(Array44d())); CALL_SUBTEST_4(array_generic( ArrayXXcf(internal::random<int>(1, EIGEN_TEST_MAX_SIZE), internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); - CALL_SUBTEST_7(array_generic( + CALL_SUBTEST_5(array_generic( ArrayXXf(internal::random<int>(1, EIGEN_TEST_MAX_SIZE), internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); CALL_SUBTEST_8(array_generic( ArrayXXi(internal::random<int>(1, EIGEN_TEST_MAX_SIZE), internal::random<int>(1, EIGEN_TEST_MAX_SIZE))));
diff --git a/test/exceptions.cpp b/test/exceptions.cpp index e3a5893..751e291 100644 --- a/test/exceptions.cpp +++ b/test/exceptions.cpp
@@ -8,7 +8,7 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. // Various sanity tests with exceptions and non trivially copyable scalar type. -// - no memory leak when a custom scalar type trow an exceptions +// - no memory leak when a custom scalar type throw an exceptions // - todo: complete the list of tests! #define EIGEN_STACK_ALLOCATION_LIMIT 100000000 @@ -21,9 +21,8 @@ AnnoyingScalar::countdown = 100; \ int before = AnnoyingScalar::instances; \ bool exception_thrown = false; \ - try { \ - OP; \ - } catch (my_exception) { \ + EIGEN_TRY { OP; } \ + EIGEN_CATCH(my_exception) { \ exception_thrown = true; \ VERIFY(AnnoyingScalar::instances == before && "memory leak detected in " && EIGEN_MAKESTRING(OP)); \ } \ @@ -35,7 +34,11 @@ typedef Eigen::Matrix<AnnoyingScalar, Dynamic, Dynamic> MatrixType; { +#if defined(EIGEN_EXCEPTIONS) && !defined(EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW) AnnoyingScalar::dont_throw = false; +#else + AnnoyingScalar::dont_throw = true; +#endif int n = 50; VectorType v0(n), v1(n); MatrixType m0(n, n), m1(n, n), m2(n, n);
diff --git a/test/gpu_test_helper.h b/test/gpu_test_helper.h index bbf766d..bc8c8d4 100644 --- a/test/gpu_test_helper.h +++ b/test/gpu_test_helper.h
@@ -4,7 +4,7 @@ #include <Eigen/Core> // Allow gpu** macros for generic tests. -#include <unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> +#include <Eigen/src/Core/util/GpuHipCudaDefines.inc> // std::tuple cannot be used on device, and there is a bug in cuda < 9.2 that // doesn't allow std::tuple to compile for host code either. In these cases,
diff --git a/test/main.h b/test/main.h index a8e951f..2288778 100644 --- a/test/main.h +++ b/test/main.h
@@ -343,7 +343,7 @@ #if !defined(EIGEN_TESTING_CONSTEXPR) && !defined(EIGEN_TESTING_PLAINOBJECT_CTOR) #define EIGEN_INTERNAL_DEBUGGING #endif -#include <Eigen/QR> // required for createRandomPIMatrixOfRank and generateRandomMatrixSvs +#include <Eigen/Core> inline void verify_impl(bool condition, const char* testname, const char* file, int line, const char* condition_as_string) { @@ -935,3 +935,7 @@ #endif #include "gpu_test_helper.h" + +#ifndef EIGEN_TEST_MAX_SIZE +#define EIGEN_TEST_MAX_SIZE 320 +#endif
diff --git a/test/maxsizevector.cpp b/test/maxsizevector.cpp index 7fe691a..82aa257 100644 --- a/test/maxsizevector.cpp +++ b/test/maxsizevector.cpp
@@ -1,6 +1,8 @@ #include "main.h" +#ifdef EIGEN_EXCEPTIONS #include <exception> // std::exception +#endif #include <Eigen/src/Core/util/MaxSizeVector.h> @@ -31,28 +33,27 @@ std::cout << '~'; --Foo::object_count; } - +#ifdef EIGEN_EXCEPTIONS class Fail : public std::exception {}; +#endif }; Index Foo::object_count = 0; Index Foo::object_limit = 0; -EIGEN_DECLARE_TEST(cxx11_maxsizevector) { +EIGEN_DECLARE_TEST(maxsizevector) { typedef MaxSizeVector<Foo> VectorX; Foo::object_count = 0; for (int r = 0; r < g_repeat; r++) { Index rows = internal::random<Index>(3, 30); Foo::object_limit = internal::random<Index>(0, rows - 2); std::cout << "object_limit = " << Foo::object_limit << std::endl; - bool exception_raised = false; #ifdef EIGEN_EXCEPTIONS + bool exception_raised = false; try { -#endif std::cout << "\nVectorX m(" << rows << ");\n"; VectorX vect(rows); for (int i = 0; i < rows; ++i) vect.push_back(Foo()); -#ifdef EIGEN_EXCEPTIONS VERIFY(false); // not reached if exceptions are enabled } catch (const Foo::Fail&) { exception_raised = true;
diff --git a/test/packet_segment.cpp b/test/packet_segment.cpp index 6fa6a29..8a5469c 100644 --- a/test/packet_segment.cpp +++ b/test/packet_segment.cpp
@@ -142,6 +142,21 @@ static void run() {} }; +template <bool Enable = internal::packet_traits<half>::Vectorizable> +void testReverseEdgeCase() { + // this reversed cast uses a non-zero offset for ploadSegment + Index size = 16 * internal::packet_traits<half>::size + 1; + VectorX<half> v1(size); + VectorX<float> v2(size), v3(size); + v1.setRandom(); + v2 = v1.reverse().cast<float>(); + v3 = v1.cast<float>().reverse(); + VERIFY_IS_EQUAL(v2, v3); +} + +template <> +void testReverseEdgeCase<false>() {} + template <typename Scalar> void test_packet_segment() { packet_segment_test_driver<Scalar, internal::packet_traits<Scalar>::size>::run(); @@ -164,5 +179,6 @@ test_packet_segment<double>(); test_packet_segment<std::complex<float>>(); test_packet_segment<std::complex<double>>(); + testReverseEdgeCase(); } }
diff --git a/test/packetmath.cpp b/test/packetmath.cpp index 4e09361..5f48d71 100644 --- a/test/packetmath.cpp +++ b/test/packetmath.cpp
@@ -354,28 +354,28 @@ for (int i = 0; i < size; ++i) { data1[i] = internal::random<Scalar>(); } - CHECK_CWISE1(internal::ptrue, internal::ptrue); + CHECK_CWISE1_MASK(internal::ptrue, internal::ptrue); CHECK_CWISE2_IF(true, internal::pandnot, internal::pandnot); for (int i = 0; i < PacketSize; ++i) { data1[i] = Scalar(RealScalar(i)); data1[i + PacketSize] = internal::random<bool>() ? data1[i] : Scalar(0); } - CHECK_CWISE2_IF(true, internal::pcmp_eq, internal::pcmp_eq); + CHECK_CWISE2_MASK(internal::pcmp_eq, internal::pcmp_eq); // Test (-0) == (0) for signed operations for (int i = 0; i < PacketSize; ++i) { data1[i] = Scalar(-0.0); data1[i + PacketSize] = internal::random<bool>() ? data1[i] : Scalar(0); } - CHECK_CWISE2_IF(true, internal::pcmp_eq, internal::pcmp_eq); + CHECK_CWISE2_MASK(internal::pcmp_eq, internal::pcmp_eq); // Test NaN for (int i = 0; i < PacketSize; ++i) { data1[i] = NumTraits<Scalar>::quiet_NaN(); data1[i + PacketSize] = internal::random<bool>() ? data1[i] : Scalar(0); } - CHECK_CWISE2_IF(true, internal::pcmp_eq, internal::pcmp_eq); + CHECK_CWISE2_MASK(internal::pcmp_eq, internal::pcmp_eq); } template <typename Scalar, typename Packet> @@ -384,28 +384,27 @@ const int size = 2 * PacketSize; EIGEN_ALIGN_MAX Scalar data1[size]; EIGEN_ALIGN_MAX Scalar data2[size]; - EIGEN_ALIGN_MAX Scalar ref[size]; for (int i = 0; i < PacketSize; ++i) { data1[i] = internal::random<Scalar>(); data1[i + PacketSize] = internal::random<bool>() ? data1[i] : Scalar(0); } - CHECK_CWISE2_IF(true, internal::pcmp_lt_or_nan, internal::pcmp_lt_or_nan); + CHECK_CWISE2_MASK(internal::pcmp_lt_or_nan, internal::pcmp_lt_or_nan); // Test (-0) <=/< (0) for signed operations for (int i = 0; i < PacketSize; ++i) { data1[i] = Scalar(-0.0); data1[i + PacketSize] = internal::random<bool>() ? data1[i] : Scalar(0); } - CHECK_CWISE2_IF(true, internal::pcmp_lt_or_nan, internal::pcmp_lt_or_nan); + CHECK_CWISE2_MASK(internal::pcmp_lt_or_nan, internal::pcmp_lt_or_nan); // Test NaN for (int i = 0; i < PacketSize; ++i) { data1[i] = NumTraits<Scalar>::quiet_NaN(); data1[i + PacketSize] = internal::random<bool>() ? data1[i] : Scalar(0); } - CHECK_CWISE2_IF(true, internal::pcmp_lt_or_nan, internal::pcmp_lt_or_nan); + CHECK_CWISE2_MASK(internal::pcmp_lt_or_nan, internal::pcmp_lt_or_nan); } template <typename Scalar, typename Packet, typename EnableIf = void> @@ -422,31 +421,30 @@ const int size = 2 * PacketSize; EIGEN_ALIGN_MAX Scalar data1[size]; EIGEN_ALIGN_MAX Scalar data2[size]; - EIGEN_ALIGN_MAX Scalar ref[size]; for (int i = 0; i < PacketSize; ++i) { data1[i] = internal::random<Scalar>(); data1[i + PacketSize] = internal::random<bool>() ? data1[i] : Scalar(0); } - CHECK_CWISE2_IF(true, internal::pcmp_le, internal::pcmp_le); - CHECK_CWISE2_IF(true, internal::pcmp_lt, internal::pcmp_lt); + CHECK_CWISE2_MASK(internal::pcmp_le, internal::pcmp_le); + CHECK_CWISE2_MASK(internal::pcmp_lt, internal::pcmp_lt); // Test (-0) <=/< (0) for signed operations for (int i = 0; i < PacketSize; ++i) { data1[i] = Scalar(-0.0); data1[i + PacketSize] = internal::random<bool>() ? data1[i] : Scalar(0); } - CHECK_CWISE2_IF(true, internal::pcmp_le, internal::pcmp_le); - CHECK_CWISE2_IF(true, internal::pcmp_lt, internal::pcmp_lt); + CHECK_CWISE2_MASK(internal::pcmp_le, internal::pcmp_le); + CHECK_CWISE2_MASK(internal::pcmp_lt, internal::pcmp_lt); // Test NaN for (int i = 0; i < PacketSize; ++i) { data1[i] = NumTraits<Scalar>::quiet_NaN(); data1[i + PacketSize] = internal::random<bool>() ? data1[i] : Scalar(0); } - CHECK_CWISE2_IF(true, internal::pcmp_le, internal::pcmp_le); - CHECK_CWISE2_IF(true, internal::pcmp_lt, internal::pcmp_lt); + CHECK_CWISE2_MASK(internal::pcmp_le, internal::pcmp_le); + CHECK_CWISE2_MASK(internal::pcmp_lt, internal::pcmp_lt); } }; @@ -700,7 +698,7 @@ for (int i = 0; i < PacketSize; ++i) { data1[i] = internal::random<Scalar>(Scalar(0) - limit, limit); } - } else if (!NumTraits<Scalar>::IsInteger && !NumTraits<Scalar>::IsComplex) { + } else if (!NumTraits<Scalar>::IsInteger && !NumTraits<Scalar>::IsComplex && !std::is_same<Scalar, bool>::value) { // Prevent very small product results by adjusting range. Otherwise, // we may end up with multiplying e.g. 32 Eigen::halfs with values < 1. for (int i = 0; i < PacketSize; ++i) {
diff --git a/test/packetmath_test_shared.h b/test/packetmath_test_shared.h index 7d7a0da..64b13e3 100644 --- a/test/packetmath_test_shared.h +++ b/test/packetmath_test_shared.h
@@ -115,6 +115,30 @@ VERIFY(test::areApprox(ref, data2, PacketSize) && #POP); \ } +#define CHECK_CWISE1_MASK(REFOP, POP) \ + { \ + bool ref_mask[PacketSize] = {}; \ + bool data_mask[PacketSize] = {}; \ + internal::pstore(data2, POP(internal::pload<Packet>(data1))); \ + for (int i = 0; i < PacketSize; ++i) { \ + ref_mask[i] = numext::is_exactly_zero(REFOP(data1[i])); \ + data_mask[i] = numext::is_exactly_zero(data2[i]); \ + } \ + VERIFY(test::areEqual(ref_mask, data_mask, PacketSize) && #POP); \ + } + +#define CHECK_CWISE2_MASK(REFOP, POP) \ + { \ + bool ref_mask[PacketSize] = {}; \ + bool data_mask[PacketSize] = {}; \ + internal::pstore(data2, POP(internal::pload<Packet>(data1), internal::pload<Packet>(data1 + PacketSize))); \ + for (int i = 0; i < PacketSize; ++i) { \ + ref_mask[i] = numext::is_exactly_zero(REFOP(data1[i], data1[i + PacketSize])); \ + data_mask[i] = numext::is_exactly_zero(data2[i]); \ + } \ + VERIFY(test::areEqual(ref_mask, data_mask, PacketSize) && #POP); \ + } + // Checks component-wise for input of size N. All of data1, data2, and ref // should have size at least ceil(N/PacketSize)*PacketSize to avoid memory // access errors.
diff --git a/test/product_selfadjoint.cpp b/test/product_selfadjoint.cpp index 8d1bda5..b61f300 100644 --- a/test/product_selfadjoint.cpp +++ b/test/product_selfadjoint.cpp
@@ -57,6 +57,10 @@ v1.tail(rows - 1) * v2.head(cols - 1).adjoint() + v2.head(cols - 1) * v1.tail(rows - 1).adjoint(); VERIFY_IS_APPROX(m2, m3.template triangularView<Lower>().toDenseMatrix()); } + + // matrix-vector + m2 = m1.template triangularView<Lower>(); + VERIFY_IS_APPROX(m1 * m4, m2.template selfadjointView<Lower>() * m4); } EIGEN_DECLARE_TEST(product_selfadjoint) {
diff --git a/test/redux.cpp b/test/redux.cpp index c9c3978..71ef535 100644 --- a/test/redux.cpp +++ b/test/redux.cpp
@@ -182,8 +182,8 @@ CALL_SUBTEST_5(matrixRedux(ArrayXX<int64_t>(rows, cols))); CALL_SUBTEST_6(matrixRedux(MatrixXcf(rows, cols))); CALL_SUBTEST_6(matrixRedux(ArrayXXcf(rows, cols))); - CALL_SUBTEST_6(matrixRedux(MatrixXcd(rows, cols))); - CALL_SUBTEST_6(matrixRedux(ArrayXXcd(rows, cols))); + CALL_SUBTEST_7(matrixRedux(MatrixXcd(rows, cols))); + CALL_SUBTEST_7(matrixRedux(ArrayXXcd(rows, cols))); } for (int i = 0; i < g_repeat; i++) { int size = internal::random<int>(1, maxsize);
diff --git a/test/sizeoverflow.cpp b/test/sizeoverflow.cpp index 66f820f..612067f 100644 --- a/test/sizeoverflow.cpp +++ b/test/sizeoverflow.cpp
@@ -9,6 +9,7 @@ #include "main.h" +#ifdef EIGEN_EXCEPTIONS #define VERIFY_THROWS_BADALLOC(a) \ { \ bool threw = false; \ @@ -19,6 +20,10 @@ } \ VERIFY(threw && "should have thrown bad_alloc: " #a); \ } +#else +// No way to catch a bad alloc - program terminates. +#define VERIFY_THROWS_BADALLOC(a) +#endif template <typename MatrixType> void triggerMatrixBadAlloc(Index rows, Index cols) {
diff --git a/test/svd_common.h b/test/svd_common.h index dd520f5..5174ade 100644 --- a/test/svd_common.h +++ b/test/svd_common.h
@@ -381,6 +381,7 @@ typedef Matrix<typename MatrixType::Scalar, RowsAtCompileTime, 1> RhsType; RhsType rhs = RhsType::Zero(input.rows()); + EIGEN_UNUSED_VARIABLE(rhs); // Only used if asserts are enabled. MatrixType m(input.rows(), input.cols()); svd_fill_random(m); @@ -410,6 +411,7 @@ enum { RowsAtCompileTime = MatrixType::RowsAtCompileTime }; typedef Matrix<typename MatrixType::Scalar, RowsAtCompileTime, 1> RhsType; RhsType rhs = RhsType::Zero(input.rows()); + EIGEN_UNUSED_VARIABLE(rhs); // Only used if asserts are enabled. MatrixType m(input.rows(), input.cols()); svd_fill_random(m);
diff --git a/test/zerosized.cpp b/test/zerosized.cpp index 7001327..2df2f29 100644 --- a/test/zerosized.cpp +++ b/test/zerosized.cpp
@@ -24,6 +24,8 @@ VERIFY_RAISES_ASSERT(m.minCoeff()); VERIFY_RAISES_ASSERT(m.maxCoeff()); Index i, j; + EIGEN_UNUSED_VARIABLE(i); // Only used if exceptions are enabled. + EIGEN_UNUSED_VARIABLE(j); VERIFY_RAISES_ASSERT(m.minCoeff(&i, &j)); VERIFY_RAISES_ASSERT(m.maxCoeff(&i, &j)); VERIFY_RAISES_ASSERT(m.reshaped().minCoeff(&i));
diff --git a/unsupported/Eigen/CXX11/Tensor b/unsupported/Eigen/CXX11/Tensor index 290a0c0..7375a9b 100644 --- a/unsupported/Eigen/CXX11/Tensor +++ b/unsupported/Eigen/CXX11/Tensor
@@ -45,7 +45,7 @@ #include <thread> #if defined(EIGEN_USE_THREADS) || defined(EIGEN_USE_SYCL) -#include "ThreadPool" +#include "../../../Eigen/ThreadPool" #endif #ifdef EIGEN_USE_GPU
diff --git a/unsupported/Eigen/CXX11/src/Tensor/README.md b/unsupported/Eigen/CXX11/src/Tensor/README.md index 395f44d..45a9454 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/README.md +++ b/unsupported/Eigen/CXX11/src/Tensor/README.md
@@ -24,12 +24,14 @@ indicating the sizes of the instance along each of the the `rank` dimensions. - // Create a tensor of rank 3 of sizes 2, 3, 4. This tensor owns - // memory to hold 24 floating point values (24 = 2 x 3 x 4). - Tensor<float, 3> t_3d(2, 3, 4); +```cpp +// Create a tensor of rank 3 of sizes 2, 3, 4. This tensor owns +// memory to hold 24 floating point values (24 = 2 x 3 x 4). +Tensor<float, 3> t_3d(2, 3, 4); - // Resize t_3d by assigning a tensor of different sizes, but same rank. - t_3d = Tensor<float, 3>(3, 4, 3); +// Resize t_3d by assigning a tensor of different sizes, but same rank. +t_3d = Tensor<float, 3>(3, 4, 3); +``` #### Constructor Tensor<data_type, rank>(size_array) @@ -38,8 +40,10 @@ `Eigen::array<Eigen::Index>`. The array can be constructed automatically from an initializer list. - // Create a tensor of strings of rank 2 with sizes 5, 7. - Tensor<string, 2> t_2d({5, 7}); +```cpp +// Create a tensor of strings of rank 2 with sizes 5, 7. +Tensor<string, 2> t_2d({5, 7}); +``` ### Class TensorFixedSize<data_type, Sizes<size0, size1, ...>> @@ -50,17 +54,19 @@ If the total number of elements in a fixed size tensor is small enough the tensor data is held onto the stack and does not cause heap allocation and free. - // Create a 4 x 3 tensor of floats. - TensorFixedSize<float, Sizes<4, 3>> t_4x3; +```cpp +// Create a 4 x 3 tensor of floats. +TensorFixedSize<float, Sizes<4, 3>> t_4x3; +``` ### Class TensorMap<Tensor<data_type, rank>> This is the class to use to create a tensor on top of memory allocated and owned by another part of your code. It allows to view any piece of allocated -memory as a Tensor. Instances of this class do not own the memory where the +memory as a `Tensor`. Instances of this class do not own the memory where the data are stored. -A TensorMap is not resizable because it does not own the memory where its data +A `TensorMap` is not resizable because it does not own the memory where its data are stored. #### Constructor TensorMap<Tensor<data_type, rank>>(data, size0, size1, ...) @@ -69,23 +75,24 @@ storage for the data, and "rank" size attributes. The storage has to be large enough to hold all the data. - // Map a tensor of ints on top of stack-allocated storage. - int storage[128]; // 2 x 4 x 2 x 8 = 128 - TensorMap<Tensor<int, 4>> t_4d(storage, 2, 4, 2, 8); +```cpp +// Map a tensor of ints on top of stack-allocated storage. +int storage[128]; // 2 x 4 x 2 x 8 = 128 +TensorMap<Tensor<int, 4>> t_4d(storage, 2, 4, 2, 8); - // The same storage can be viewed as a different tensor. - // You can also pass the sizes as an array. - TensorMap<Tensor<int, 2>> t_2d(storage, 16, 8); +// The same storage can be viewed as a different tensor. +// You can also pass the sizes as an array. +TensorMap<Tensor<int, 2>> t_2d(storage, 16, 8); - // You can also map fixed-size tensors. Here we get a 1d view of - // the 2d fixed-size tensor. - TensorFixedSize<float, Sizes<4, 3>> t_4x3; - TensorMap<Tensor<float, 1>> t_12(t_4x3.data(), 12); - +// You can also map fixed-size tensors. Here we get a 1d view of +// the 2d fixed-size tensor. +TensorFixedSize<float, Sizes<4, 3>> t_4x3; +TensorMap<Tensor<float, 1>> t_12(t_4x3.data(), 12); +``` #### Class TensorRef -See Assigning to a `TensorRef` below. +See **Assigning to a `TensorRef`**. ## Accessing Tensor Elements @@ -96,24 +103,25 @@ The expression can be used as an l-value to set the value of the element at the specified position. The value returned is of the datatype of the tensor. - // Set the value of the element at position (0, 1, 0); - Tensor<float, 3> t_3d(2, 3, 4); - t_3d(0, 1, 0) = 12.0f; +```cpp +// Set the value of the element at position (0, 1, 0); +Tensor<float, 3> t_3d(2, 3, 4); +t_3d(0, 1, 0) = 12.0f; - // Initialize all elements to random values. - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 3; ++j) { - for (int k = 0; k < 4; ++k) { - t_3d(i, j, k) = ...some random value...; - } - } +// Initialize all elements to random values. +for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 3; ++j) { + for (int k = 0; k < 4; ++k) { + t_3d(i, j, k) = ...some random value...; } + } +} - // Print elements of a tensor. - for (int i = 0; i < 2; ++i) { - LOG(INFO) << t_3d(i, 0, 0); - } - +// Print elements of a tensor. +for (int i = 0; i < 2; ++i) { + std::cout << t_3d(i, 0, 0); +} +``` ## TensorLayout @@ -123,8 +131,10 @@ The layout of a tensor is optionally specified as part of its type. If not specified explicitly column major is assumed. - Tensor<float, 3, ColMajor> col_major; // equivalent to Tensor<float, 3> - TensorMap<Tensor<float, 3, RowMajor> > row_major(data, ...); +```cpp +Tensor<float, 3, ColMajor> col_major; // equivalent to Tensor<float, 3> +TensorMap<Tensor<float, 3, RowMajor> > row_major(data, ...); +``` All the arguments to an expression must use the same layout. Attempting to mix different layouts will result in a compilation error. @@ -133,47 +143,50 @@ `swap_layout()` method. Note that this will also reverse the order of the dimensions. - Tensor<float, 2, ColMajor> col_major(2, 4); - Tensor<float, 2, RowMajor> row_major(2, 4); +```cpp +Tensor<float, 2, ColMajor> col_major(2, 4); +Tensor<float, 2, RowMajor> row_major(2, 4); - Tensor<float, 2> col_major_result = col_major; // ok, layouts match - Tensor<float, 2> col_major_result = row_major; // will not compile +Tensor<float, 2> col_major_result = col_major; // ok, layouts match +Tensor<float, 2> col_major_result = row_major; // will not compile - // Simple layout swap - col_major_result = row_major.swap_layout(); - eigen_assert(col_major_result.dimension(0) == 4); - eigen_assert(col_major_result.dimension(1) == 2); +// Simple layout swap +col_major_result = row_major.swap_layout(); +eigen_assert(col_major_result.dimension(0) == 4); +eigen_assert(col_major_result.dimension(1) == 2); - // Swap the layout and preserve the order of the dimensions - array<int, 2> shuffle(1, 0); - col_major_result = row_major.swap_layout().shuffle(shuffle); - eigen_assert(col_major_result.dimension(0) == 2); - eigen_assert(col_major_result.dimension(1) == 4); - +// Swap the layout and preserve the order of the dimensions +array<int, 2> shuffle(1, 0); +col_major_result = row_major.swap_layout().shuffle(shuffle); +eigen_assert(col_major_result.dimension(0) == 2); +eigen_assert(col_major_result.dimension(1) == 4); +``` ## Tensor Operations The Eigen Tensor library provides a vast library of operations on Tensors: numerical operations such as addition and multiplication, geometry operations such as slicing and shuffling, etc. These operations are available as methods -of the Tensor classes, and in some cases as operator overloads. For example +of the `Tensor` classes, and in some cases as operator overloads. For example the following code computes the elementwise addition of two tensors: - Tensor<float, 3> t1(2, 3, 4); - ...set some values in t1... - Tensor<float, 3> t2(2, 3, 4); - ...set some values in t2... - // Set t3 to the element wise sum of t1 and t2 - Tensor<float, 3> t3 = t1 + t2; +```cpp +Tensor<float, 3> t1(2, 3, 4); +t2.setRandom(); +Tensor<float, 3> t2(2, 3, 4); +t2.setRandom(); +// Set t3 to the element wise sum of t1 and t2 +Tensor<float, 3> t3 = t1 + t2; +``` While the code above looks easy enough, it is important to understand that the expression `t1 + t2` is not actually adding the values of the tensors. The expression instead constructs a "tensor operator" object of the class -TensorCwiseBinaryOp<scalar_sum>, which has references to the tensors +`TensorCwiseBinaryOp<scalar_sum>`, which has references to the tensors `t1` and `t2`. This is a small C++ object that knows how to add `t1` and `t2`. It is only when the value of the expression is assigned to the tensor `t3` that the addition is actually performed. Technically, -this happens through the overloading of `operator=()` in the Tensor class. +this happens through the overloading of `operator=` in the Tensor class. This mechanism for computing tensor expressions allows for lazy evaluation and optimizations which are what make the tensor library very fast. @@ -181,16 +194,19 @@ Of course, the tensor operators do nest, and the expression `t1 + t2 * 0.3f` is actually represented with the (approximate) tree of operators: - TensorCwiseBinaryOp<scalar_sum>(t1, TensorCwiseUnaryOp<scalar_mul>(t2, 0.3f)) - +```cpp +TensorCwiseBinaryOp<scalar_sum>(t1, TensorCwiseUnaryOp<scalar_mul>(t2, 0.3f)) +``` ### Tensor Operations and C++ "auto" -Because Tensor operations create tensor operators, the C++ `auto` keyword +Because `Tensor` operations create tensor operators, the C++ `auto` keyword does not have its intuitive meaning. Consider these 2 lines of code: - Tensor<float, 3> t3 = t1 + t2; - auto t4 = t1 + t2; +```cpp +Tensor<float, 3> t3 = t1 + t2; +auto t4 = t1 + t2; +``` In the first line we allocate the tensor `t3` and it will contain the result of the addition of `t1` and `t2`. In the second line, `t4` @@ -198,191 +214,221 @@ `t1` and `t2`. In fact, `t4` is *not* a tensor and you cannot get the values of its elements: - Tensor<float, 3> t3 = t1 + t2; - cout << t3(0, 0, 0); // OK prints the value of t1(0, 0, 0) + t2(0, 0, 0) +```cpp +Tensor<float, 3> t3 = t1 + t2; +std::cout << t3(0, 0, 0); // OK prints the value of t1(0, 0, 0) + t2(0, 0, 0) - auto t4 = t1 + t2; - cout << t4(0, 0, 0); // Compilation error! +auto t4 = t1 + t2; +std::cout << t4(0, 0, 0); // Compilation error! +``` -When you use `auto` you do not get a Tensor as a result but instead a -non-evaluated expression. So only use `auto` to delay evaluation. +When you use `auto` you do not get a `Tensor` as a result but instead a +non-evaluated expression. +So only use `auto` to delay evaluation. Unfortunately, there is no single underlying concrete type for holding -non-evaluated expressions, hence you have to use auto in the case when you do +non-evaluated expressions, hence you have to use `auto` in the case when you do want to hold non-evaluated expressions. When you need the results of set of tensor computations you have to assign the -result to a Tensor that will be capable of holding onto them. This can be -either a normal Tensor, a fixed size Tensor, or a TensorMap on an existing +result to a `Tensor` that will be capable of holding onto them. This can be +either a normal `Tensor`, a `TensorFixedSize`, or a `TensorMap` on an existing piece of memory. All the following will work: - auto t4 = t1 + t2; +```cpp +auto t4 = t1 + t2; - Tensor<float, 3> result = t4; // Could also be: result(t4); - cout << result(0, 0, 0); +Tensor<float, 3> result = t4; // Could also be: result(t4); +std::cout << result(0, 0, 0); - TensorMap<float, 4> result(<a float* with enough space>, <size0>, ...) = t4; - cout << result(0, 0, 0); +TensorMap<float, 4> result(<a float* with enough space>, <size0>, ...) = t4; +std::cout << result(0, 0, 0); - TensorFixedSize<float, Sizes<size0, ...>> result = t4; - cout << result(0, 0, 0); +TensorFixedSize<float, Sizes<size0, ...>> result = t4; +std::cout << result(0, 0, 0); +``` Until you need the results, you can keep the operation around, and even reuse it for additional operations. As long as you keep the expression as an operation, no computation is performed. - // One way to compute exp((t1 + t2) * 0.2f); - auto t3 = t1 + t2; - auto t4 = t3 * 0.2f; - auto t5 = t4.exp(); - Tensor<float, 3> result = t5; +```cpp +// One way to compute exp((t1 + t2) * 0.2f); +auto t3 = t1 + t2; +auto t4 = t3 * 0.2f; +auto t5 = t4.exp(); +Tensor<float, 3> result = t5; - // Another way, exactly as efficient as the previous one: - Tensor<float, 3> result = ((t1 + t2) * 0.2f).exp(); +// Another way, exactly as efficient as the previous one: +Tensor<float, 3> result = ((t1 + t2) * 0.2f).exp(); +``` ### Controlling When Expression are Evaluated There are several ways to control when expressions are evaluated: -* Assignment to a Tensor, TensorFixedSize, or TensorMap. -* Use of the eval() method. -* Assignment to a TensorRef. +* Assignment to a `Tensor`, `TensorFixedSize`, or `TensorMap`. +* Use of the `eval()` method. +* Assignment to a `TensorRef`. #### Assigning to a Tensor, TensorFixedSize, or TensorMap. -The most common way to evaluate an expression is to assign it to a Tensor. In -the example below, the `auto` declarations make the intermediate values +The most common way to evaluate an expression is to assign it to a `Tensor`. +In the example below, the `auto` declarations make the intermediate values "Operations", not Tensors, and do not cause the expressions to be evaluated. The assignment to the Tensor `result` causes the evaluation of all the operations. - auto t3 = t1 + t2; // t3 is an Operation. - auto t4 = t3 * 0.2f; // t4 is an Operation. - auto t5 = t4.exp(); // t5 is an Operation. - Tensor<float, 3> result = t5; // The operations are evaluated. +```cpp +auto t3 = t1 + t2; // t3 is an Operation. +auto t4 = t3 * 0.2f; // t4 is an Operation. +auto t5 = t4.exp(); // t5 is an Operation. +Tensor<float, 3> result = t5; // The operations are evaluated. +``` If you know the ranks and sizes of the Operation value you can assign the -Operation to a TensorFixedSize instead of a Tensor, which is a bit more -efficient. +Operation to a `TensorFixedSize` instead of a `Tensor`, which is a bit more efficient. - // We know that the result is a 4x4x2 tensor! - TensorFixedSize<float, Sizes<4, 4, 2>> result = t5; +```cpp +// We know that the result is a 4x4x2 tensor! +TensorFixedSize<float, Sizes<4, 4, 2>> result = t5; +``` -Simiarly, assigning an expression to a TensorMap causes its evaluation. Like -tensors of type TensorFixedSize, TensorMaps cannot be resized so they have to +Similarly, assigning an expression to a `TensorMap` causes its evaluation. +Like tensors of type `TensorFixedSize`, a `TensorMap` cannot be resized so they have to have the rank and sizes of the expression that are assigned to them. #### Calling eval(). When you compute large composite expressions, you sometimes want to tell Eigen that an intermediate value in the expression tree is worth evaluating ahead of -time. This is done by inserting a call to the `eval()` method of the +time. +This is done by inserting a call to the `eval()` method of the expression Operation. - // The previous example could have been written: - Tensor<float, 3> result = ((t1 + t2) * 0.2f).exp(); +```cpp +// The previous example could have been written: +Tensor<float, 3> result = ((t1 + t2) * 0.2f).exp(); - // If you want to compute (t1 + t2) once ahead of time you can write: - Tensor<float, 3> result = ((t1 + t2).eval() * 0.2f).exp(); +// If you want to compute (t1 + t2) once ahead of time you can write: +Tensor<float, 3> result = ((t1 + t2).eval() * 0.2f).exp(); +``` Semantically, calling `eval()` is equivalent to materializing the value of -the expression in a temporary Tensor of the right size. The code above in -effect does: +the expression in a temporary `Tensor` of the right size. +The code above in effect does: - // .eval() knows the size! - TensorFixedSize<float, Sizes<4, 4, 2>> tmp = t1 + t2; - Tensor<float, 3> result = (tmp * 0.2f).exp(); +```cpp +// .eval() knows the size! +TensorFixedSize<float, Sizes<4, 4, 2>> tmp = t1 + t2; +Tensor<float, 3> result = (tmp * 0.2f).exp(); +``` Note that the return value of `eval()` is itself an Operation, so the following code does not do what you may think: - // Here t3 is an evaluation Operation. t3 has not been evaluated yet. - auto t3 = (t1 + t2).eval(); +```cpp +// Here t3 is an evaluation Operation. t3 has not been evaluated yet. +auto t3 = (t1 + t2).eval(); - // You can use t3 in another expression. Still no evaluation. - auto t4 = (t3 * 0.2f).exp(); +// You can use t3 in another expression. Still no evaluation. +auto t4 = (t3 * 0.2f).exp(); - // The value is evaluated when you assign the Operation to a Tensor, using - // an intermediate tensor to represent t3.x - Tensor<float, 3> result = t4; +// The value is evaluated when you assign the Operation to a Tensor, using +// an intermediate tensor to represent t3.x +Tensor<float, 3> result = t4; +``` While in the examples above calling `eval()` does not make a difference in performance, in other cases it can make a huge difference. In the expression below the `broadcast()` expression causes the `X.maximum()` expression to be evaluated many times: - Tensor<...> X ...; - Tensor<...> Y = ((X - X.maximum(depth_dim).reshape(dims2d).broadcast(bcast)) - * beta).exp(); +```cpp +Tensor<...> X ...; +Tensor<...> Y = ((X - X.maximum(depth_dim).reshape(dims2d).broadcast(bcast)) + * beta).exp(); +``` Inserting a call to `eval()` between the `maximum()` and -`reshape()` calls guarantees that maximum() is only computed once and +`reshape()` calls guarantees that `maximum()` is only computed once and greatly speeds-up execution: - Tensor<...> Y = - ((X - X.maximum(depth_dim).eval().reshape(dims2d).broadcast(bcast)) - * beta).exp(); +```cpp +Tensor<...> Y = + ((X - X.maximum(depth_dim).eval().reshape(dims2d).broadcast(bcast)) + * beta).exp(); +``` -In the other example below, the tensor `Y` is both used in the expression -and its assignment. This is an aliasing problem and if the evaluation is not -done in the right order Y will be updated incrementally during the evaluation +In the other example below, the tensor `Y` is both used in the expression and its assignment. +This is an aliasing problem and if the evaluation is not done in the right order +Y will be updated incrementally during the evaluation resulting in bogus results: - Tensor<...> Y ...; - Y = Y / (Y.sum(depth_dim).reshape(dims2d).broadcast(bcast)); +```cpp + Tensor<...> Y ...; + Y = Y / (Y.sum(depth_dim).reshape(dims2d).broadcast(bcast)); +``` Inserting a call to `eval()` between the `sum()` and `reshape()` expressions ensures that the sum is computed before any updates to `Y` are done. - Y = Y / (Y.sum(depth_dim).eval().reshape(dims2d).broadcast(bcast)); +```cpp + Y = Y / (Y.sum(depth_dim).eval().reshape(dims2d).broadcast(bcast)); +``` Note that an eval around the full right hand side expression is not needed -because the generated has to compute the i-th value of the right hand side +because the generated has to compute the `i`-th value of the right hand side before assigning it to the left hand side. However, if you were assigning the expression value to a shuffle of `Y` then you would need to force an eval for correctness by adding an `eval()` call for the right hand side: - Y.shuffle(...) = - (Y / (Y.sum(depth_dim).eval().reshape(dims2d).broadcast(bcast))).eval(); - +```cpp + Y.shuffle(...) = + (Y / (Y.sum(depth_dim).eval().reshape(dims2d).broadcast(bcast))).eval(); +``` #### Assigning to a TensorRef. If you need to access only a few elements from the value of an expression you -can avoid materializing the value in a full tensor by using a TensorRef. +can avoid materializing the value in a full tensor by using a `TensorRef`. -A TensorRef is a small wrapper class for any Eigen Operation. It provides +A `TensorRef` is a small wrapper class for any Eigen Operation. It provides overloads for the `()` operator that let you access individual values in -the expression. TensorRef is convenient, because the Operation themselves do +the expression. +`TensorRef` is convenient, because the Operation themselves do not provide a way to access individual elements. - // Create a TensorRef for the expression. The expression is not - // evaluated yet. - TensorRef<Tensor<float, 3> > ref = ((t1 + t2) * 0.2f).exp(); +```cpp +// Create a TensorRef for the expression. The expression is not +// evaluated yet. +TensorRef<Tensor<float, 3> > ref = ((t1 + t2) * 0.2f).exp(); - // Use "ref" to access individual elements. The expression is evaluated - // on the fly. - float at_0 = ref(0, 0, 0); - cout << ref(0, 1, 0); +// Use "ref" to access individual elements. The expression is evaluated +// on the fly. +float at_0 = ref(0, 0, 0); +std::cout << ref(0, 1, 0); +``` -Only use TensorRef when you need a subset of the values of the expression. -TensorRef only computes the values you access. However note that if you are -going to access all the values it will be much faster to materialize the -results in a Tensor first. +Only use `TensorRef` when you need a subset of the values of the expression. +`TensorRef` only computes the values you access. +However note that if you are going to access all the values it will be much + faster to materialize the results in a `Tensor` first. -In some cases, if the full Tensor result would be very large, you may save -memory by accessing it as a TensorRef. But not always. So don't count on it. +In some cases, if the full `Tensor` result would be very large, you may save +memory by accessing it as a `TensorRef`. +But not always. +So don't count on it. ### Controlling How Expressions Are Evaluated The tensor library provides several implementations of the various operations such as contractions and convolutions. The implementations are optimized for -different environments: single threaded on CPU, multi threaded on CPU, or on a -GPU using cuda. Additional implementations may be added later. +different environments: single threaded on CPU, multi threaded on CPU, or on a GPU using cuda. You can choose which implementation to use with the `device()` call. If you do not choose an implementation explicitly the default implementation that @@ -396,43 +442,51 @@ For example, the following code adds two tensors using the default single-threaded CPU implementation: - Tensor<float, 2> a(30, 40); - Tensor<float, 2> b(30, 40); - Tensor<float, 2> c = a + b; +```cpp +Tensor<float, 2> a(30, 40); +Tensor<float, 2> b(30, 40); +Tensor<float, 2> c = a + b; +``` To choose a different implementation you have to insert a `device()` call before the assignment of the result. For technical C++ reasons this requires -that the Tensor for the result be declared on its own. This means that you -have to know the size of the result. +that the `Tensor` for the result be declared on its own. +This means that you have to know the size of the result. - Eigen::Tensor<float, 2> c(30, 40); - c.device(...) = a + b; +```cpp +Eigen::Tensor<float, 2> c(30, 40); +c.device(...) = a + b; +``` The call to `device()` must be the last call on the left of the operator=. You must pass to the `device()` call an Eigen device object. There are -presently three devices you can use: DefaultDevice, ThreadPoolDevice and -GpuDevice. +presently three devices you can use: `DefaultDevice`, `ThreadPoolDevice` and +`GpuDevice`. #### Evaluating With the DefaultDevice This is exactly the same as not inserting a `device()` call. - DefaultDevice my_device; - c.device(my_device) = a + b; +```cpp +DefaultDevice my_device; +c.device(my_device) = a + b; +``` #### Evaluating with a Thread Pool - // Create the Eigen ThreadPool - Eigen::ThreadPool pool(8 /* number of threads in pool */) +```cpp +// Create the Eigen ThreadPool +Eigen::ThreadPool pool(8 /* number of threads in pool */) - // Create the Eigen ThreadPoolDevice. - Eigen::ThreadPoolDevice my_device(&pool, 4 /* number of threads to use */); +// Create the Eigen ThreadPoolDevice. +Eigen::ThreadPoolDevice my_device(&pool, 4 /* number of threads to use */); - // Now just use the device when evaluating expressions. - Eigen::Tensor<float, 2> c(30, 50); - c.device(my_device) = a.contract(b, dot_product_dims); +// Now just use the device when evaluating expressions. +Eigen::Tensor<float, 2> c(30, 50); +c.device(my_device) = a.contract(b, dot_product_dims); +``` #### Evaluating On GPU @@ -451,7 +505,7 @@ #### <Tensor-Type>::Dimensions -Acts like an array of ints. Has an `int size` attribute, and can be +Acts like an array of `int`. Has an `int size` attribute, and can be indexed like an array to access individual values. Used to represent the dimensions of a tensor. See `dimensions()`. @@ -463,8 +517,7 @@ #### <Tensor-Type>::Scalar Represents the datatype of individual tensor elements. For example, for a -`Tensor<float>`, `Scalar` is the type `float`. See -`setConstant()`. +`Tensor<float>`, `Scalar` is the type `float`. See `setConstant()`. #### (Operation) @@ -473,8 +526,8 @@ Operation returns after evaluation. The Operation will have to be evaluated, for example by assigning it to a -tensor, before you can access the values of the resulting tensor. You can also -access the values through a TensorRef. +`Tensor`, before you can access the values of the resulting tensor. You can also +access the values through a `TensorRef`. ## Built-in Tensor Methods @@ -482,71 +535,80 @@ These are usual C++ methods that act on tensors immediately. They are not Operations which provide delayed evaluation of their results. Unless specified otherwise, all the methods listed below are available on all tensor classes: -Tensor, TensorFixedSize, and TensorMap. +`Tensor`, `TensorFixedSize`, and `TensorMap`. ## Metadata ### int NumDimensions -Constant value indicating the number of dimensions of a Tensor. This is also -known as the tensor "rank". +Constant value indicating the number of dimensions of a `Tensor`. +This is also known as the tensor rank. - Eigen::Tensor<float, 2> a(3, 4); - cout << "Dims " << a.NumDimensions; - => Dims 2 +```cpp +Eigen::Tensor<float, 2> a(3, 4); +std::cout << "Dims " << a.NumDimensions; +// Dims 2 +``` ### Dimensions dimensions() Returns an array-like object representing the dimensions of the tensor. -The actual type of the `dimensions()` result is `<Tensor-Type>::``Dimensions`. +The actual type of the `dimensions()` result is `<Tensor-Type>::Dimensions`. - Eigen::Tensor<float, 2> a(3, 4); - const Eigen::Tensor<float, 2>::Dimensions& d = a.dimensions(); - cout << "Dim size: " << d.size << ", dim 0: " << d[0] - << ", dim 1: " << d[1]; - => Dim size: 2, dim 0: 3, dim 1: 4 +```cpp +Eigen::Tensor<float, 2> a(3, 4); +const Eigen::Tensor<float, 2>::Dimensions& d = a.dimensions(); +std::cout << "Dim size: " << d.size << ", dim 0: " << d[0] + << ", dim 1: " << d[1]; +// Dim size: 2, dim 0: 3, dim 1: 4 +``` If you use a C++11 compiler, you can use `auto` to simplify the code: - const auto& d = a.dimensions(); - cout << "Dim size: " << d.size << ", dim 0: " << d[0] - << ", dim 1: " << d[1]; - => Dim size: 2, dim 0: 3, dim 1: 4 +```cpp +const auto& d = a.dimensions(); +std::cout << "Dim size: " << d.size << ", dim 0: " << d[0] + << ", dim 1: " << d[1]; +// Dim size: 2, dim 0: 3, dim 1: 4 +``` ### Index dimension(Index n) Returns the n-th dimension of the tensor. The actual type of the -`dimension()` result is `<Tensor-Type>::``Index`, but you can +`dimension()` result is `<Tensor-Type>::Index`, but you can always use it like an int. - Eigen::Tensor<float, 2> a(3, 4); - int dim1 = a.dimension(1); - cout << "Dim 1: " << dim1; - => Dim 1: 4 +```cpp +Eigen::Tensor<float, 2> a(3, 4); +int dim1 = a.dimension(1); +std::cout << "Dim 1: " << dim1; +// Dim 1: 4 +``` ### Index size() Returns the total number of elements in the tensor. This is the product of all the tensor dimensions. The actual type of the `size()` result is -`<Tensor-Type>::``Index`, but you can always use it like an int. +`<Tensor-Type>::Index`, but you can always use it like an int. - Eigen::Tensor<float, 2> a(3, 4); - cout << "Size: " << a.size(); - => Size: 12 - +```cpp +Eigen::Tensor<float, 2> a(3, 4); +std::cout << "Size: " << a.size(); +/// Size: 12 +``` ### Getting Dimensions From An Operation A few operations provide `dimensions()` directly, e.g. `TensorReslicingOp`. Most operations defer calculating dimensions until the operation is being evaluated. If you need access to the dimensions -of a deferred operation, you can wrap it in a TensorRef (see Assigning to a -TensorRef above), which provides `dimensions()` and `dimension()` as -above. +of a deferred operation, you can wrap it in a `TensorRef` (see +**Assigning to a TensorRef** above), which provides +`dimensions()` and `dimension()` as above. -TensorRef can also wrap the plain Tensor types, so this is a useful idiom in -templated contexts where the underlying object could be either a raw Tensor -or some deferred operation (e.g. a slice of a Tensor). In this case, the +`TensorRef` can also wrap the plain `Tensor` types, so this is a useful idiom in +templated contexts where the underlying object could be either a raw `Tensor` +or some deferred operation (e.g. a slice of a `Tensor`). In this case, the template code can wrap the object in a TensorRef and reason about its dimensionality while remaining agnostic to the underlying type. @@ -558,41 +620,46 @@ Creates a tensor of the specified size. The number of arguments must be equal to the rank of the tensor. The content of the tensor is not initialized. - Eigen::Tensor<float, 2> a(3, 4); - cout << "NumRows: " << a.dimension(0) << " NumCols: " << a.dimension(1) << endl; - => NumRows: 3 NumCols: 4 - +```cpp +Eigen::Tensor<float, 2> a(3, 4); +std::cout << "NumRows: " << a.dimension(0) << " NumCols: " << a.dimension(1) << endl; +// NumRows: 3 NumCols: 4 +``` ### TensorFixedSize -Creates a tensor of the specified size. The number of arguments in the Sizes<> +Creates a tensor of the specified size. The number of arguments in the `Sizes<>` template parameter determines the rank of the tensor. The content of the tensor is not initialized. - Eigen::TensorFixedSize<float, Sizes<3, 4>> a; - cout << "Rank: " << a.rank() << endl; - => Rank: 2 - cout << "NumRows: " << a.dimension(0) << " NumCols: " << a.dimension(1) << endl; - => NumRows: 3 NumCols: 4 +```cpp +Eigen::TensorFixedSize<float, Sizes<3, 4>> a; +std::cout << "Rank: " << a.rank() << endl; +// Rank: 2 +std::cout << "NumRows: " << a.dimension(0) + << " NumCols: " << a.dimension(1) << endl; +// NumRows: 3 NumCols: 4 +``` ### TensorMap Creates a tensor mapping an existing array of data. The data must not be freed -until the TensorMap is discarded, and the size of the data must be large enough +until the `TensorMap` is discarded, and the size of the data must be large enough to accommodate the coefficients of the tensor. - float data[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; - Eigen::TensorMap<Tensor<float, 2>> a(data, 3, 4); - cout << "NumRows: " << a.dimension(0) << " NumCols: " << a.dimension(1) << endl; - => NumRows: 3 NumCols: 4 - cout << "a(1, 2): " << a(1, 2) << endl; - => a(1, 2): 7 - +```cpp +float data[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; +Eigen::TensorMap<Tensor<float, 2>> a(data, 3, 4); +std::cout << "NumRows: " << a.dimension(0) << " NumCols: " << a.dimension(1) << endl; +// NumRows: 3 NumCols: 4 +std::cout << "a(1, 2): " << a(1, 2) << endl; +// a(1, 2): 7 +``` ## Contents Initialization -When a new Tensor or a new TensorFixedSize are created, memory is allocated to +When a new `Tensor` or a new `TensorFixedSize` are created, memory is allocated to hold all the tensor elements, but the memory is not initialized. Similarly, -when a new TensorMap is created on top of non-initialized memory the memory its +when a new `TensorMap` is created on top of non-initialized memory the memory its contents are not initialized. You can use one of the methods below to initialize the tensor memory. These @@ -607,39 +674,42 @@ Returns the tensor itself in case you want to chain another call. - a.setConstant(12.3f); - cout << "Constant: " << endl << a << endl << endl; - => - Constant: - 12.3 12.3 12.3 12.3 - 12.3 12.3 12.3 12.3 - 12.3 12.3 12.3 12.3 +```cpp +a.setConstant(12.3f); +std::cout << "Constant: " << endl << a << endl << endl; +// Constant: +// 12.3 12.3 12.3 12.3 +// 12.3 12.3 12.3 12.3 +// 12.3 12.3 12.3 12.3 +``` Note that `setConstant()` can be used on any tensor where the element type has a copy constructor and an `operator=()`: - Eigen::Tensor<string, 2> a(2, 3); - a.setConstant("yolo"); - cout << "String tensor: " << endl << a << endl << endl; - => - String tensor: - yolo yolo yolo - yolo yolo yolo +```cpp +Eigen::Tensor<string, 2> a(2, 3); +a.setConstant("yolo"); +std::cout << "String tensor: " << endl << a << endl << endl; +// String tensor: +// yolo yolo yolo +// yolo yolo yolo +``` ### <Tensor-Type> setZero() Fills the tensor with zeros. Equivalent to `setConstant(Scalar(0))`. Returns the tensor itself in case you want to chain another call. - a.setZero(); - cout << "Zeros: " << endl << a << endl << endl; - => - Zeros: - 0 0 0 0 - 0 0 0 0 - 0 0 0 0 +```cpp +a.setZero(); +std::cout << "Zeros: " << endl << a << endl << endl; +// Zeros: +// 0 0 0 0 +// 0 0 0 0 +// 0 0 0 0 +``` ### <Tensor-Type> setValues({..initializer_list}) @@ -647,7 +717,7 @@ The type of the initializer list depends on the type and rank of the tensor. If the tensor has rank N, the initializer list must be nested N times. The -most deeply nested lists must contains P scalars of the Tensor type where P is +most deeply nested lists must contains P scalars of the `Tensor` type where P is the size of the last dimension of the Tensor. For example, for a `TensorFixedSize<float, 2, 3>` the initializer list must @@ -656,120 +726,129 @@ `setValues()` returns the tensor itself in case you want to chain another call. - Eigen::Tensor<float, 2> a(2, 3); - a.setValues({{0.0f, 1.0f, 2.0f}, {3.0f, 4.0f, 5.0f}}); - cout << "a" << endl << a << endl << endl; - => - a - 0 1 2 - 3 4 5 +```cpp +Eigen::Tensor<float, 2> a(2, 3); +a.setValues({{0.0f, 1.0f, 2.0f}, {3.0f, 4.0f, 5.0f}}); +std::cout << "a" << endl << a << endl << endl; + +// a +// 0 1 2 +// 3 4 5 +``` If a list is too short, the corresponding elements of the tensor will not be changed. This is valid at each level of nesting. For example the following code only sets the values of the first row of the tensor. - Eigen::Tensor<int, 2> a(2, 3); - a.setConstant(1000); - a.setValues({{10, 20, 30}}); - cout << "a" << endl << a << endl << endl; - => - a - 10 20 30 - 1000 1000 1000 +```cpp +Eigen::Tensor<int, 2> a(2, 3); +a.setConstant(1000); +a.setValues({{10, 20, 30}}); +std::cout << "a" << endl << a << endl << endl; +// a +// 10 20 30 +// 1000 1000 1000 +``` ### <Tensor-Type> setRandom() Fills the tensor with random values. Returns the tensor itself in case you want to chain another call. - a.setRandom(); - cout << "Random: " << endl << a << endl << endl; - => - Random: - 0.680375 0.59688 -0.329554 0.10794 - -0.211234 0.823295 0.536459 -0.0452059 - 0.566198 -0.604897 -0.444451 0.257742 +```cpp +a.setRandom(); +std::cout << "Random: " << endl << a << endl << endl; +// Random: +// 0.680375 0.59688 -0.329554 0.10794 +// -0.211234 0.823295 0.536459 -0.0452059 +// 0.566198 -0.604897 -0.444451 0.257742 +``` You can customize `setRandom()` by providing your own random number generator as a template argument: - a.setRandom<MyRandomGenerator>(); +```cpp +a.setRandom<MyRandomGenerator>(); +``` Here, `MyRandomGenerator` must be a struct with the following member -functions, where Scalar and Index are the same as `<Tensor-Type>::``Scalar` -and `<Tensor-Type>::``Index`. +functions, where Scalar and Index are the same as `<Tensor-Type>::Scalar` +and `<Tensor-Type>::Index`. See `struct UniformRandomGenerator` in TensorFunctors.h for an example. - // Custom number generator for use with setRandom(). - struct MyRandomGenerator { - // Default and copy constructors. Both are needed - MyRandomGenerator() { } - MyRandomGenerator(const MyRandomGenerator& ) { } +```cpp +// Custom number generator for use with setRandom(). +struct MyRandomGenerator { + // Default and copy constructors. Both are needed + MyRandomGenerator() { } + MyRandomGenerator(const MyRandomGenerator& ) { } - // Return a random value to be used. "element_location" is the - // location of the entry to set in the tensor, it can typically - // be ignored. - Scalar operator()(Eigen::DenseIndex element_location, - Eigen::DenseIndex /*unused*/ = 0) const { - return <randomly generated value of type T>; - } + // Return a random value to be used. "element_location" is the + // location of the entry to set in the tensor, it can typically + // be ignored. + Scalar operator()(Eigen::DenseIndex element_location, + Eigen::DenseIndex /*unused*/ = 0) const { + return <randomly generated value of type T>; + } - // Same as above but generates several numbers at a time. - typename internal::packet_traits<Scalar>::type packetOp( - Eigen::DenseIndex packet_location, Eigen::DenseIndex /*unused*/ = 0) const { - return <a packet of randomly generated values>; - } - }; + // Same as above but generates several numbers at a time. + typename internal::packet_traits<Scalar>::type packetOp( + Eigen::DenseIndex packet_location, Eigen::DenseIndex /*unused*/ = 0) const { + return <a packet of randomly generated values>; + } +}; +``` You can also use one of the 2 random number generators that are part of the tensor library: * UniformRandomGenerator * NormalRandomGenerator - ## Data Access The Tensor, TensorFixedSize, and TensorRef classes provide the following accessors to access the tensor coefficients: - const Scalar& operator()(const array<Index, NumIndices>& indices) - const Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) - Scalar& operator()(const array<Index, NumIndices>& indices) - Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) +```cpp +const Scalar& operator()(const array<Index, NumIndices>& indices) +const Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) +Scalar& operator()(const array<Index, NumIndices>& indices) +Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) +``` The number of indices must be equal to the rank of the tensor. Moreover, these accessors are not available on tensor expressions. In order to access the values of a tensor expression, the expression must either be evaluated or wrapped in a TensorRef. - ### Scalar* data() and const Scalar* data() const Returns a pointer to the storage for the tensor. The pointer is const if the tensor was const. This allows direct access to the data. The layout of the -data depends on the tensor layout: RowMajor or ColMajor. +data depends on the tensor layout: `RowMajor` or `ColMajor`. This access is usually only needed for special cases, for example when mixing Eigen Tensor code with other libraries. Scalar is the type of data stored in the tensor. - Eigen::Tensor<float, 2> a(3, 4); - float* a_data = a.data(); - a_data[0] = 123.45f; - cout << "a(0, 0): " << a(0, 0); - => a(0, 0): 123.45 - +```cpp +Eigen::Tensor<float, 2> a(3, 4); +float* a_data = a.data(); +a_data[0] = 123.45f; +std::cout << "a(0, 0): " << a(0, 0); +// a(0, 0): 123.45 +``` ## Tensor Operations All the methods documented below return non evaluated tensor `Operations`. -These can be chained: you can apply another Tensor Operation to the value +These can be chained: you can apply another `Tensor` Operation to the value returned by the method. The chain of Operation is evaluated lazily, typically when it is assigned to a -tensor. See "Controlling when Expression are Evaluated" for more details about +tensor. See **Controlling When Expression are Evaluated** for more details about their evaluation. ### (Operation) constant(const Scalar& val) @@ -779,26 +858,29 @@ This is useful, for example, when you want to add or subtract a constant from a tensor, or multiply every element of a tensor by a scalar. +However, such operations can also be performed using operator overloads (see `operator+`). - Eigen::Tensor<float, 2> a(2, 3); - a.setConstant(1.0f); - Eigen::Tensor<float, 2> b = a + a.constant(2.0f); - Eigen::Tensor<float, 2> c = b * b.constant(0.2f); - cout << "a" << endl << a << endl << endl; - cout << "b" << endl << b << endl << endl; - cout << "c" << endl << c << endl << endl; - => - a - 1 1 1 - 1 1 1 - b - 3 3 3 - 3 3 3 +```cpp +Eigen::Tensor<float, 2> a(2, 3); +a.setConstant(1.0f); +Eigen::Tensor<float, 2> b = a + a.constant(2.0f); +Eigen::Tensor<float, 2> c = b * b.constant(0.2f); +std::cout << "a" << endl << a << endl << endl; +std::cout << "b" << endl << b << endl << endl; +std::cout << "c" << endl << c << endl << endl; +// a +// 1 1 1 +// 1 1 1 - c - 0.6 0.6 0.6 - 0.6 0.6 0.6 +// b +// 3 3 3 +// 3 3 3 + +// c +// 0.6 0.6 0.6 +// 0.6 0.6 0.6 +``` ### (Operation) random() @@ -809,20 +891,20 @@ The generation of random values can be customized in the same manner as for `setRandom()`. - Eigen::Tensor<float, 2> a(2, 3); - a.setConstant(1.0f); - Eigen::Tensor<float, 2> b = a + a.random(); - cout << "a" << endl << a << endl << endl; - cout << "b" << endl << b << endl << endl; - => - a - 1 1 1 - 1 1 1 +```cpp +Eigen::Tensor<float, 2> a(2, 3); +a.setConstant(1.0f); +Eigen::Tensor<float, 2> b = a + a.random(); +std::cout << "a\n" << a << "\n\n"; +std::cout << "b\n" << b << "\n\n"; - b - 1.68038 1.5662 1.82329 - 0.788766 1.59688 0.395103 - +// a +// 1 1 1 +// 1 1 1 +// b +// 1.68038 1.5662 1.82329 +// 0.788766 1.59688 +``` ## Unary Element Wise Operations @@ -835,19 +917,21 @@ Returns a tensor of the same type and dimensions as the original tensor containing the opposite values of the original tensor. - Eigen::Tensor<float, 2> a(2, 3); - a.setConstant(1.0f); - Eigen::Tensor<float, 2> b = -a; - cout << "a" << endl << a << endl << endl; - cout << "b" << endl << b << endl << endl; - => - a - 1 1 1 - 1 1 1 +```cpp +Eigen::Tensor<float, 2> a(2, 3); +a.setConstant(1.0f); +Eigen::Tensor<float, 2> b = -a; +std::cout << "a\n" << a << "\n\n"; +std::cout << "b\n" << b << "\n\n"; - b - -1 -1 -1 - -1 -1 -1 +// a +// 1 1 1 +// 1 1 1 +// +// b +// -1 -1 -1 +// -1 -1 -1 +``` ### (Operation) sqrt() @@ -894,12 +978,14 @@ Returns a tensor with the same dimensions as the original tensor containing the real part of the complex values of the original tensor. +The result has a real-valued scalar type. ### (Operation) imag() Returns a tensor with the same dimensions as the original tensor containing the imaginary part of the complex values of the original tensor. +The result has a real-valued scalar type. ### (Operation) pow(Scalar exponent) @@ -911,35 +997,158 @@ tensor coefficients. For example, only integer exponents can be used in conjunction with tensors of integer values. -You can use cast() to lift this restriction. For example this computes +You can use `cast()` to lift this restriction. For example this computes cubic roots of an int Tensor: - Eigen::Tensor<int, 2> a(2, 3); - a.setValues({{0, 1, 8}, {27, 64, 125}}); - Eigen::Tensor<double, 2> b = a.cast<double>().pow(1.0 / 3.0); - cout << "a" << endl << a << endl << endl; - cout << "b" << endl << b << endl << endl; - => - a - 0 1 8 - 27 64 125 +```cpp +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{0, 1, 8}, {27, 64, 125}}); +Eigen::Tensor<double, 2> b = a.cast<double>().pow(1.0 / 3.0); +std::cout << "a" << endl << a << endl << endl; +std::cout << "b" << endl << b << endl << endl; - b - 0 1 2 - 3 4 5 +// a +// 0 1 8 +// 27 64 125 +// +// b +// 0 1 2 +// 3 4 5 +``` -### (Operation) operator * (Scalar scale) +### (Operation) operator* (Scalar s) -Multiplies all the coefficients of the input tensor by the provided scale. +Multiplies every element of the input tensor by the scalar `s`: +```cpp +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{1, 2, 3}, + {4, 5, 6}}); +Eigen::Tensor<int,2> scaled_a = a * 2; + +std::cout << "a\n" << a << "\n"; +std::cout << "scaled_a\n" << scaled_a << "\n"; + +// a +// 1 2 3 +// 4 5 6 +// +// scaled_a +// 2 4 6 +// 8 10 12 +``` +### (Operation) operator+ (Scalar s) +Adds `s` to every element in the tensor. + +### (Operation) operator- (Scalar s) +Subtracts `s` from every element in the tensor. + +### (Operation) operator/ (Scalar s) +Divides every element in the tensor by `s`. + +### (Operation) operator% (Scalar s) +Computes the element-wise modulus (remainder) of each tensor element divided by `s` + +**Only integer types are supported.** +For floating-point tensors, implement a `unaryExpr` using `std::fmod`. ### (Operation) cwiseMax(Scalar threshold) -TODO +Returns the coefficient-wise maximum between two tensors. +```cpp +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{0, 100, 200}, {300, 400, 500}}); +Eigen::Tensor<int, 2> b(2, 3); +b.setValues({{-1, -2, 300}, {-4, 555, -6}}); + +Eigen::Tensor<int, 2> c = a.cwiseMax(b); + +std::cout << "a\n" << a << "\n" + << "b\n" << b << "\n" + << "c\n" << c << "\n"; + +// a +// 0 100 200 +// 300 400 500 + +// b +// -1 -2 300 +// -4 555 -6 + +// c +// 0 100 300 +// 300 555 500 +``` ### (Operation) cwiseMin(Scalar threshold) -TODO +Returns the coefficient-wise minimum between two tensors. + +```cpp +Eigen::Tensor<int, 2> a(2, 2); +a.setValues({{0, 100}, {300, -900}}); + +Eigen::Tensor<int, 2> b(2, 2); +b.setValues({{-1, -2}, {400, 555}}); + +Eigen::Tensor<int, 2> c = a.cwiseMin(b); + +std::cout << "a\n" << a << "\n" + << "b\n" << b << "\n" + << "c\n" << c << "\n"; + +// a +// 0 100 +// 300 -900 + +// b +// -1 -2 +// 400 555 + +// c +// -1 -2 +// 300 -900 +``` ### (Operation) unaryExpr(const CustomUnaryOp& func) -TODO +Applies a user defined function to each element in the tensor. +Supports lambdas or functor structs with an operator(). + +Using lambda: +```cpp +Eigen::Tensor<float, 2> a(2, 3); +a.setValues({{0, -.5, -1}, {.5, 1.5, 2.0}}); +auto my_func = [](float el){ return std::abs(el + 0.5f);}; +Eigen::Tensor<float, 2> b = a.unaryExpr(my_func); +std::cout << "a\n" << a << "\n" + << "b\n" << b << "\n"; +=> +a + 0 -0.5 -1 +0.5 1.5 2 +b +0.5 0 0.5 + 1 2 2.5 +``` + +Using a functor to normalize and clamp values to `[-1.0, 1.0]`: + +```cpp +template<typename Scalar> +struct NormalizedClamp { +NormalizedClamp(Scalar lo, Scalar hi) : _lo(lo), _hi(hi) {} +Scalar operator()(Scalar x) const { + if (x < _lo) return Scalar(0); + if (x > _hi) return Scalar(1); + return (x - _lo) / (_hi - _lo); +} +Scalar _lo, _hi; +}; + +Eigen::Tensor<float, 2> c = a.unaryExpr(NormalizedClamp<float>(-1.0f, 1.0f)); +std::cout << "c\n" << c << "\n"; + +// c +// 0.5 0.25 0 +// 0.75 1 1 +``` ## Binary Element Wise Operations @@ -984,7 +1193,7 @@ ### (Operation) Logical operators -The following logical operators are supported as well: +The following boolean operators are supported: * `operator&&(const OtherDerived& other)` * `operator||(const OtherDerived& other)` @@ -995,18 +1204,25 @@ * `operator==(const OtherDerived& other)` * `operator!=(const OtherDerived& other)` -They all return a tensor of boolean values. + as well as bitwise operators: + * `operator&(const OtherDerived& other)` + * `operator|(const OtherDerived& other)` + * `operator^(const OtherDerived& other)` + +The resulting tensor retains the input scalar type. ## Selection (select(const ThenDerived& thenTensor, const ElseDerived& elseTensor) Selection is a coefficient-wise ternary operator that is the tensor equivalent to the if-then-else operation. +```cpp Tensor<bool, 3> if = ...; Tensor<float, 3> then = ...; Tensor<float, 3> else = ...; Tensor<float, 3> result = if.select(then, else); +``` The 3 arguments must be of the same dimensions, which will also be the dimension of the result. The 'if' tensor must be of type boolean, the 'then' and the @@ -1023,27 +1239,29 @@ Tensor *contractions* are a generalization of the matrix product to the multidimensional case. - // Create 2 matrices using tensors of rank 2 - Eigen::Tensor<int, 2> a(2, 3); - a.setValues({{1, 2, 3}, {6, 5, 4}}); - Eigen::Tensor<int, 2> b(3, 2); - b.setValues({{1, 2}, {4, 5}, {5, 6}}); +```cpp +// Create 2 matrices using tensors of rank 2 +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{1, 2, 3}, {6, 5, 4}}); +Eigen::Tensor<int, 2> b(3, 2); +b.setValues({{1, 2}, {4, 5}, {5, 6}}); - // Compute the traditional matrix product - Eigen::array<Eigen::IndexPair<int>, 1> product_dims = { Eigen::IndexPair<int>(1, 0) }; - Eigen::Tensor<int, 2> AB = a.contract(b, product_dims); +// Compute the traditional matrix product +Eigen::array<Eigen::IndexPair<int>, 1> product_dims = { Eigen::IndexPair<int>(1, 0) }; +Eigen::Tensor<int, 2> AB = a.contract(b, product_dims); - // Compute the product of the transpose of the matrices - Eigen::array<Eigen::IndexPair<int>, 1> transposed_product_dims = { Eigen::IndexPair<int>(0, 1) }; - Eigen::Tensor<int, 2> AtBt = a.contract(b, transposed_product_dims); +// Compute the product of the transpose of the matrices +Eigen::array<Eigen::IndexPair<int>, 1> transposed_product_dims = { Eigen::IndexPair<int>(0, 1) }; +Eigen::Tensor<int, 2> AtBt = a.contract(b, transposed_product_dims); - // Contraction to scalar value using a double contraction. - // First coordinate of both tensors are contracted as well as both second coordinates, i.e., this computes the sum of the squares of the elements. - Eigen::array<Eigen::IndexPair<int>, 2> double_contraction_product_dims = { Eigen::IndexPair<int>(0, 0), Eigen::IndexPair<int>(1, 1) }; - Eigen::Tensor<int, 0> AdoubleContractedA = a.contract(a, double_contraction_product_dims); +// Contraction to scalar value using a double contraction. +// First coordinate of both tensors are contracted as well as both second coordinates, i.e., this computes the sum of the squares of the elements. +Eigen::array<Eigen::IndexPair<int>, 2> double_contraction_product_dims = { Eigen::IndexPair<int>(0, 0), Eigen::IndexPair<int>(1, 1) }; +Eigen::Tensor<int, 0> AdoubleContractedA = a.contract(a, double_contraction_product_dims); - // Extracting the scalar value of the tensor contraction for further usage - int value = AdoubleContractedA(0); +// Extracting the scalar value of the tensor contraction for further usage +int value = AdoubleContractedA(0); +``` ## Reduction Operations @@ -1074,116 +1292,165 @@ increasing order. Example: Reduction along one dimension. +```cpp +// Create a tensor of 2 dimensions +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{1, 2, 3}, {6, 5, 4}}); +// Reduce it along the second dimension (1)... +Eigen::array<int, 1> dims({1 /* dimension to reduce */}); +// ...using the "maximum" operator. +// The result is a tensor with one dimension. The size of +// that dimension is the same as the first (non-reduced) dimension of a. +Eigen::Tensor<int, 1> b = a.maximum(dims); +std::cout << "a" << endl << a << endl << endl; +std::cout << "b" << endl << b << endl << endl; - // Create a tensor of 2 dimensions - Eigen::Tensor<int, 2> a(2, 3); - a.setValues({{1, 2, 3}, {6, 5, 4}}); - // Reduce it along the second dimension (1)... - Eigen::array<int, 1> dims({1 /* dimension to reduce */}); - // ...using the "maximum" operator. - // The result is a tensor with one dimension. The size of - // that dimension is the same as the first (non-reduced) dimension of a. - Eigen::Tensor<int, 1> b = a.maximum(dims); - cout << "a" << endl << a << endl << endl; - cout << "b" << endl << b << endl << endl; - => - a - 1 2 3 - 6 5 4 +// a +// 1 2 3 +// 6 5 4 - b - 3 - 6 - +// b +// 3 +// 6 +``` Example: Reduction along two dimensions. +```cpp +Eigen::Tensor<float, 3, Eigen::ColMajor> a(2, 3, 4); +a.setValues({{{0.0f, 1.0f, 2.0f, 3.0f}, + {7.0f, 6.0f, 5.0f, 4.0f}, + {8.0f, 9.0f, 10.0f, 11.0f}}, + {{12.0f, 13.0f, 14.0f, 15.0f}, + {19.0f, 18.0f, 17.0f, 16.0f}, + {20.0f, 21.0f, 22.0f, 23.0f}}}); +// The tensor a has 3 dimensions. We reduce along the +// first 2, resulting in a tensor with a single dimension +// of size 4 (the last dimension of a.) +// Note that we pass the array of reduction dimensions +// directly to the maximum() call. +Eigen::Tensor<float, 1, Eigen::ColMajor> b = + a.maximum(Eigen::array<int, 2>({0, 1})); +std::cout << "b" << endl << b << endl << endl; - Eigen::Tensor<float, 3, Eigen::ColMajor> a(2, 3, 4); - a.setValues({{{0.0f, 1.0f, 2.0f, 3.0f}, - {7.0f, 6.0f, 5.0f, 4.0f}, - {8.0f, 9.0f, 10.0f, 11.0f}}, - {{12.0f, 13.0f, 14.0f, 15.0f}, - {19.0f, 18.0f, 17.0f, 16.0f}, - {20.0f, 21.0f, 22.0f, 23.0f}}}); - // The tensor a has 3 dimensions. We reduce along the - // first 2, resulting in a tensor with a single dimension - // of size 4 (the last dimension of a.) - // Note that we pass the array of reduction dimensions - // directly to the maximum() call. - Eigen::Tensor<float, 1, Eigen::ColMajor> b = - a.maximum(Eigen::array<int, 2>({0, 1})); - cout << "b" << endl << b << endl << endl; - => - b - 20 - 21 - 22 - 23 - +// b +// 20 +// 21 +// 22 +// 23 +``` #### Reduction along all dimensions As a special case, if you pass no parameter to a reduction operation the original tensor is reduced along *all* its dimensions. The result is a scalar, represented as a zero-dimension tensor. - Eigen::Tensor<float, 3> a(2, 3, 4); - a.setValues({{{0.0f, 1.0f, 2.0f, 3.0f}, - {7.0f, 6.0f, 5.0f, 4.0f}, - {8.0f, 9.0f, 10.0f, 11.0f}}, - {{12.0f, 13.0f, 14.0f, 15.0f}, - {19.0f, 18.0f, 17.0f, 16.0f}, - {20.0f, 21.0f, 22.0f, 23.0f}}}); - // Reduce along all dimensions using the sum() operator. - Eigen::Tensor<float, 0> b = a.sum(); - cout << "b" << endl << b << endl << endl; - => - b - 276 +```cpp +Eigen::Tensor<float, 3> a(2, 3, 4); +a.setValues({{{0.0f, 1.0f, 2.0f, 3.0f}, + {7.0f, 6.0f, 5.0f, 4.0f}, + {8.0f, 9.0f, 10.0f, 11.0f}}, + {{12.0f, 13.0f, 14.0f, 15.0f}, + {19.0f, 18.0f, 17.0f, 16.0f}, + {20.0f, 21.0f, 22.0f, 23.0f}}}); +// Reduce along all dimensions using the sum() operator. +Eigen::Tensor<float, 0> b = a.sum(); +std::cout << "b\n" << b; +// b +// 276 +``` +You can extract the scalar directly by casting the expression and extract the first and only coefficient: +```cpp +float sum = static_cast<Eigen::Tensor<float, 0>>(a.sum())(); +``` -### (Operation) sum(const Dimensions& new_dims) +### (Operation) sum(const Dimensions& reduction_dims) ### (Operation) sum() -Reduce a tensor using the sum() operator. The resulting values +Reduce a tensor using the `sum()` operator. The resulting values are the sum of the reduced values. -### (Operation) mean(const Dimensions& new_dims) +### (Operation) mean(const Dimensions& reduction_dims) ### (Operation) mean() -Reduce a tensor using the mean() operator. The resulting values +Reduce a tensor using the `mean()` operator. The resulting values are the mean of the reduced values. -### (Operation) maximum(const Dimensions& new_dims) +### (Operation) maximum(const Dimensions& reduction_dims) ### (Operation) maximum() -Reduce a tensor using the maximum() operator. The resulting values are the +Reduce a tensor using the `maximum()` operator. The resulting values are the largest of the reduced values. -### (Operation) minimum(const Dimensions& new_dims) +### (Operation) minimum(const Dimensions& reduction_dims) ### (Operation) minimum() -Reduce a tensor using the minimum() operator. The resulting values +Reduce a tensor using the `minimum()` operator. The resulting values are the smallest of the reduced values. -### (Operation) prod(const Dimensions& new_dims) +### (Operation) prod(const Dimensions& reduction_dims) ### (Operation) prod() -Reduce a tensor using the prod() operator. The resulting values +Reduce a tensor using the `prod()` operator. The resulting values are the product of the reduced values. -### (Operation) all(const Dimensions& new_dims) +### (Operation) all(const Dimensions& reduction_dims) ### (Operation) all() -Reduce a tensor using the all() operator. Casts tensor to bool and then checks +Reduce a tensor using the `all()` operator. Casts tensor to bool and then checks whether all elements are true. Runs through all elements rather than short-circuiting, so may be significantly inefficient. -### (Operation) any(const Dimensions& new_dims) +### (Operation) any(const Dimensions& reduction_dims) ### (Operation) any() -Reduce a tensor using the any() operator. Casts tensor to bool and then checks +Reduce a tensor using the `any()` operator. Casts tensor to bool and then checks whether any element is true. Runs through all elements rather than short-circuiting, so may be significantly inefficient. -### (Operation) reduce(const Dimensions& new_dims, const Reducer& reducer) +### (Operation) argmax(const Dimensions& reduction_dim) +### (Operation) argmax() + +Reduce a tensor using the `argmax()` operator. + +The resulting values are the indices of the largest elements along the specified dimension. + +Only a single `reduction_dim` is supported. + +If multiple elements share the maximum value, the one with the **lowest index** is returned. + +```cpp +Eigen::Tensor<float, 2> a(2, 3); +a.setValues({{1, 4, 8}, {3, 4, 2}}); + +Eigen::Tensor<Eigen::Index, 1> argmax_dim0 = a.argmax(0); + +std::cout << "a:\n" << a << "\n"; +for (int i = 0; i < argmax_dim0.size(); ++i) { + std::cout << "argmax along dim 0 at index " << i << " = " << argmax_dim0(i) << "\n"; +} + +// a: +// 1 4 8 +// 3 4 2 +// argmax along dim 0 at index 0 = 1 +// argmax along dim 0 at index 1 = 0 +// argmax along dim 0 at index 2 = 0 +``` + + To compute the index of the global maximum, use the overload without arguments (which flattens the tensor). + + +```cpp +Eigen::Tensor<Eigen::Index, 0> argmax_flat = a.argmax(); +std::cout << "Flat argmax index: " << argmax_flat(); + +// Flat argmax index: 4 +``` + +### (Operation) argmin(const Dimensions& reduction_dim) +### (Operation) argmin() +See `argmax`. + +### (Operation) reduce(const Dimensions& reduction_dims, const Reducer& reducer) Reduce a tensor using a user-defined reduction operator. See `SumReducer` in TensorFunctors.h for information on how to implement a reduction operator. @@ -1201,23 +1468,24 @@ Example: Trace along 2 dimensions. - // Create a tensor of 3 dimensions - Eigen::Tensor<int, 3> a(2, 2, 3); - a.setValues({{{1, 2, 3}, {4, 5, 6}}, {{7, 8, 9}, {10, 11, 12}}}); - // Specify the dimensions along which the trace will be computed. - // In this example, the trace can only be computed along the dimensions - // with indices 0 and 1 - Eigen::array<int, 2> dims({0, 1}); - // The output tensor contains all but the trace dimensions. - Tensor<int, 1> a_trace = a.trace(dims); - cout << "a_trace:" << endl; - cout << a_trace << endl; - => - a_trace: - 11 - 13 - 15 +```cpp +// Create a tensor of 3 dimensions +Eigen::Tensor<int, 3> a(2, 2, 3); +a.setValues({{{1, 2, 3}, {4, 5, 6}}, {{7, 8, 9}, {10, 11, 12}}}); +// Specify the dimensions along which the trace will be computed. +// In this example, the trace can only be computed along the dimensions +// with indices 0 and 1 +Eigen::array<int, 2> dims({0, 1}); +// The output tensor contains all but the trace dimensions. +Tensor<int, 1> a_trace = a.trace(dims); +std::cout << "a_trace:" << endl; +std::cout << a_trace << endl; +// a_trace: +// 11 +// 13 +// 15 +``` ### (Operation) trace(const Dimensions& new_dims) ### (Operation) trace() @@ -1227,19 +1495,20 @@ Example: Trace along all dimensions. - // Create a tensor of 3 dimensions, with all dimensions having the same size. - Eigen::Tensor<int, 3> a(3, 3, 3); - a.setValues({{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}, - {{10, 11, 12}, {13, 14, 15}, {16, 17, 18}}, - {{19, 20, 21}, {22, 23, 24}, {25, 26, 27}}}); - // Result is a zero dimension tensor - Tensor<int, 0> a_trace = a.trace(); - cout<<"a_trace:"<<endl; - cout<<a_trace<<endl; - => - a_trace: - 42 +```cpp +// Create a tensor of 3 dimensions, with all dimensions having the same size. +Eigen::Tensor<int, 3> a(3, 3, 3); +a.setValues({{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}, + {{10, 11, 12}, {13, 14, 15}, {16, 17, 18}}, + {{19, 20, 21}, {22, 23, 24}, {25, 26, 27}}}); +// Result is a zero dimension tensor +Tensor<int, 0> a_trace = a.trace(); +std::cout<<"a_trace:"<<endl; +std::cout<<a_trace<<endl; +// a_trace: +// 42 +``` ## Scan Operations @@ -1251,24 +1520,26 @@ prefix sum of the tensor along the given axis. Example: -dd a comment to this line +Cumulative sum along the second dimension - // Create a tensor of 2 dimensions - Eigen::Tensor<int, 2> a(2, 3); - a.setValues({{1, 2, 3}, {4, 5, 6}}); - // Scan it along the second dimension (1) using summation - Eigen::Tensor<int, 2> b = a.cumsum(1); - // The result is a tensor with the same size as the input - cout << "a" << endl << a << endl << endl; - cout << "b" << endl << b << endl << endl; - => - a - 1 2 3 - 4 5 6 +```cpp +// Create a tensor of 2 dimensions +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{1, 2, 3}, {4, 5, 6}}); +// Scan it along the second dimension (1) using summation +Eigen::Tensor<int, 2> b = a.cumsum(1); +// The result is a tensor with the same size as the input +std::cout << "a" << endl << a << endl << endl; +std::cout << "b" << endl << b << endl << endl; - b - 1 3 6 - 4 9 15 +// a +// 1 2 3 +// 4 5 6 + +// b +// 1 3 6 +// 4 9 15 +``` ### (Operation) cumsum(const Index& axis) @@ -1278,7 +1549,6 @@ Perform a scan by multiplying consecutive entries. - ## Convolutions ### (Operation) convolve(const Kernel& kernel, const Dimensions& dims) @@ -1286,265 +1556,367 @@ Returns a tensor that is the output of the convolution of the input tensor with the kernel, along the specified dimensions of the input tensor. The dimension size for dimensions of the output tensor which were part of the convolution will be reduced by the formula: -output_dim_size = input_dim_size - kernel_dim_size + 1 (requires: input_dim_size >= kernel_dim_size). +```cpp +output_dim_size = input_dim_size - kernel_dim_size + 1 // (requires: input_dim_size >= kernel_dim_size). +``` The dimension sizes for dimensions that were not part of the convolution will remain the same. Performance of the convolution can depend on the length of the stride(s) of the input tensor dimension(s) along which the -convolution is computed (the first dimension has the shortest stride for ColMajor, whereas RowMajor's shortest stride is +convolution is computed (the first dimension has the shortest stride for `ColMajor`, whereas `RowMajor`'s shortest stride is for the last dimension). - // Compute convolution along the second and third dimension. - Tensor<float, 4, DataLayout> input(3, 3, 7, 11); - Tensor<float, 2, DataLayout> kernel(2, 2); - Tensor<float, 4, DataLayout> output(3, 2, 6, 11); - input.setRandom(); - kernel.setRandom(); +```cpp +// Compute convolution along the second and third dimension. +Tensor<float, 4, DataLayout> input(3, 3, 7, 11); +Tensor<float, 2, DataLayout> kernel(2, 2); +Tensor<float, 4, DataLayout> output(3, 2, 6, 11); +input.setRandom(); +kernel.setRandom(); - Eigen::array<ptrdiff_t, 2> dims({1, 2}); // Specify second and third dimension for convolution. - output = input.convolve(kernel, dims); +Eigen::array<ptrdiff_t, 2> dims({1, 2}); // Specify second and third dimension for convolution. +output = input.convolve(kernel, dims); - for (int i = 0; i < 3; ++i) { - for (int j = 0; j < 2; ++j) { - for (int k = 0; k < 6; ++k) { - for (int l = 0; l < 11; ++l) { - const float result = output(i,j,k,l); - const float expected = input(i,j+0,k+0,l) * kernel(0,0) + - input(i,j+1,k+0,l) * kernel(1,0) + - input(i,j+0,k+1,l) * kernel(0,1) + - input(i,j+1,k+1,l) * kernel(1,1); - VERIFY_IS_APPROX(result, expected); - } - } +for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 2; ++j) { + for (int k = 0; k < 6; ++k) { + for (int l = 0; l < 11; ++l) { + const float result = output(i,j,k,l); + const float expected = input(i,j+0,k+0,l) * kernel(0,0) + + input(i,j+1,k+0,l) * kernel(1,0) + + input(i,j+0,k+1,l) * kernel(0,1) + + input(i,j+1,k+1,l) * kernel(1,1); + VERIFY_IS_APPROX(result, expected); } } - + } +} +``` ## Geometrical Operations -These operations return a Tensor with different dimensions than the original -Tensor. They can be used to access slices of tensors, see them with different +These operations return a `Tensor` with different dimensions than the original +`Tensor`. They can be used to access slices of tensors, see them with different dimensions, or pad tensors with additional data. ### (Operation) reshape(const Dimensions& new_dims) Returns a view of the input tensor that has been reshaped to the specified -new dimensions. The argument new_dims is an array of Index values. The -rank of the resulting tensor is equal to the number of elements in new_dims. +new dimensions. + +The argument `new_dims` is an array of Index values. + +The rank of the resulting tensor is equal to the number of elements in `new_dims`. The product of all the sizes in the new dimension array must be equal to the number of elements in the input tensor. - // Increase the rank of the input tensor by introducing a new dimension - // of size 1. - Tensor<float, 2> input(7, 11); - array<int, 3> three_dims{{7, 11, 1}}; - Tensor<float, 3> result = input.reshape(three_dims); +```cpp +// Increase the rank of the input tensor by introducing a new dimension +// of size 1. +Tensor<float, 2> input(7, 11); +array<int, 3> three_dims{{7, 11, 1}}; +Tensor<float, 3> result = input.reshape(three_dims); - // Decrease the rank of the input tensor by merging 2 dimensions; - array<int, 1> one_dim{{7 * 11}}; - Tensor<float, 1> result = input.reshape(one_dim); +// Decrease the rank of the input tensor by merging 2 dimensions; +array<int, 1> one_dim{{7 * 11}}; +Tensor<float, 1> result = input.reshape(one_dim); +``` This operation does not move any data in the input tensor, so the resulting -contents of a reshaped Tensor depend on the data layout of the original Tensor. +contents of a reshaped `Tensor` depend on the data layout of the original `Tensor`. -For example this is what happens when you `reshape()` a 2D ColMajor tensor +For example this is what happens when you `reshape()` a 2D `ColMajor` tensor to one dimension: - Eigen::Tensor<float, 2, Eigen::ColMajor> a(2, 3); - a.setValues({{0.0f, 100.0f, 200.0f}, {300.0f, 400.0f, 500.0f}}); - Eigen::array<Eigen::DenseIndex, 1> one_dim({3 * 2}); - Eigen::Tensor<float, 1, Eigen::ColMajor> b = a.reshape(one_dim); - cout << "b" << endl << b << endl; - => - b - 0 - 300 - 100 - 400 - 200 - 500 +```cpp +Eigen::Tensor<float, 2, Eigen::ColMajor> a(2, 3); +a.setValues({{0.0f, 100.0f, 200.0f}, {300.0f, 400.0f, 500.0f}}); +Eigen::array<Eigen::DenseIndex, 1> one_dim({3 * 2}); +Eigen::Tensor<float, 1, Eigen::ColMajor> b = a.reshape(one_dim); +std::cout << "b" << endl << b << endl; -This is what happens when the 2D Tensor is RowMajor: +// b +// 0 +// 300 +// 100 +// 400 +// 200 +// 500 +``` - Eigen::Tensor<float, 2, Eigen::RowMajor> a(2, 3); - a.setValues({{0.0f, 100.0f, 200.0f}, {300.0f, 400.0f, 500.0f}}); - Eigen::array<Eigen::DenseIndex, 1> one_dim({3 * 2}); - Eigen::Tensor<float, 1, Eigen::RowMajor> b = a.reshape(one_dim); - cout << "b" << endl << b << endl; - => - b - 0 - 100 - 200 - 300 - 400 - 500 +This is what happens when the 2D `Tensor` is `RowMajor`: + +```cpp +Eigen::Tensor<float, 2, Eigen::RowMajor> a(2, 3); +a.setValues({{0.0f, 100.0f, 200.0f}, {300.0f, 400.0f, 500.0f}}); +Eigen::array<Eigen::DenseIndex, 1> one_dim({3 * 2}); +Eigen::Tensor<float, 1, Eigen::RowMajor> b = a.reshape(one_dim); +std::cout << "b" << endl << b << endl; + +// b +// 0 +// 100 +// 200 +// 300 +// 400 +// 500 +``` The reshape operation is a lvalue. In other words, it can be used on the left side of the assignment operator. The previous example can be rewritten as follow: - Eigen::Tensor<float, 2, Eigen::ColMajor> a(2, 3); - a.setValues({{0.0f, 100.0f, 200.0f}, {300.0f, 400.0f, 500.0f}}); - Eigen::array<Eigen::DenseIndex, 2> two_dim({2, 3}); - Eigen::Tensor<float, 1, Eigen::ColMajor> b(6); - b.reshape(two_dim) = a; - cout << "b" << endl << b << endl; - => - b - 0 - 300 - 100 - 400 - 200 - 500 +```cpp +Eigen::Tensor<float, 2, Eigen::ColMajor> a(2, 3); +a.setValues({{0.0f, 100.0f, 200.0f}, {300.0f, 400.0f, 500.0f}}); +Eigen::array<Eigen::DenseIndex, 2> two_dim({2, 3}); +Eigen::Tensor<float, 1, Eigen::ColMajor> b(6); +b.reshape(two_dim) = a; +std::cout << "b" << endl << b << endl; + +// b +// 0 +// 300 +// 100 +// 400 +// 200 +// 500 +``` Note that "b" itself was not reshaped but that instead the assignment is done to the reshape view of b. - ### (Operation) shuffle(const Shuffle& shuffle) -Returns a copy of the input tensor whose dimensions have been -reordered according to the specified permutation. The argument shuffle -is an array of Index values. Its size is the rank of the input -tensor. It must contain a permutation of 0, 1, ..., rank - 1. The i-th -dimension of the output tensor equals to the size of the shuffle[i]-th -dimension of the input tensor. For example: +Returns a view of the input tensor whose dimensions have been +reordered according to the specified permutation. - // Shuffle all dimensions to the left by 1. - Tensor<float, 3> input(20, 30, 50); - // ... set some values in input. - Tensor<float, 3> output = input.shuffle({1, 2, 0}) +The argument `shuffle` is an array of `Index` values: +* Its size is the rank of the input tensor. +* It must contain a permutation of `[0, 1, ..., rank - 1]`. +* The `i`-th dimension of the output tensor corresponds to the size of the dimension at position `shuffle[i]` in the input tensor. For example: - eigen_assert(output.dimension(0) == 30); - eigen_assert(output.dimension(1) == 50); - eigen_assert(output.dimension(2) == 20); +```cpp +// Shuffle all dimensions to the left by 1. +Tensor<float, 3> input(20, 30, 50); +// ... set some values in input. +Tensor<float, 3> output = input.shuffle({1, 2, 0}); -Indices into the output tensor are shuffled accordingly to formulate -indices into the input tensor. For example, one can assert in the above -code snippet that: +eigen_assert(output.dimension(0) == 30); +eigen_assert(output.dimension(1) == 50); +eigen_assert(output.dimension(2) == 20); - eigen_assert(output(3, 7, 11) == input(11, 3, 7)); +// Indices into the output tensor are shuffled accordingly to formulate +// indices into the input tensor. +eigen_assert(output(3, 7, 11) == input(11, 3, 7)); -In general, one can assert that - - eigen_assert(output(..., indices[shuffle[i]], ...) == - input(..., indices[i], ...)) +// In general: +eigen_assert(output(..., indices[shuffle[i]], ...) == + input(..., indices[i], ...)); +``` The shuffle operation results in a lvalue, which means that it can be assigned to. In other words, it can be used on the left side of the assignment operator. Let's rewrite the previous example to take advantage of this feature: - // Shuffle all dimensions to the left by 1. - Tensor<float, 3> input(20, 30, 50); - // ... set some values in input. - Tensor<float, 3> output(30, 50, 20); - output.shuffle({2, 0, 1}) = input; - +```cpp +// Shuffle all dimensions to the left by 1. +Tensor<float, 3> input(20, 30, 50); +input.setRandom(); +Tensor<float, 3> output(30, 50, 20); +output.shuffle({2, 0, 1}) = input; +``` ### (Operation) stride(const Strides& strides) Returns a view of the input tensor that strides (skips stride-1 -elements) along each of the dimensions. The argument strides is an -array of Index values. The dimensions of the resulting tensor are -ceil(input_dimensions[i] / strides[i]). +elements) along each of the dimensions. + +The argument strides is an array of `Index` values: +* Its size is the rank of the input tensor. +* Must be >= 1 + + The dimensions of the resulting tensor are `ceil(input_dimensions[i] / strides[i])`. For example this is what happens when you `stride()` a 2D tensor: - Eigen::Tensor<int, 2> a(4, 3); - a.setValues({{0, 100, 200}, {300, 400, 500}, {600, 700, 800}, {900, 1000, 1100}}); - Eigen::array<Eigen::DenseIndex, 2> strides({3, 2}); - Eigen::Tensor<int, 2> b = a.stride(strides); - cout << "b" << endl << b << endl; - => - b - 0 200 - 900 1100 +```cpp +Eigen::Tensor<int, 2> a(4, 3); +a.setValues({{0, 100, 200}, + {300, 400, 500}, + {600, 700, 800}, + {900, 1000, 1100}}); +Eigen::array<Eigen::DenseIndex, 2> strides({3, 2}); +Eigen::Tensor<int, 2> b = a.stride(strides); +std::cout << "b" << endl << b << endl; +// b +// 0 200 +// 900 1100 +``` It is possible to assign a tensor to a stride: - Tensor<float, 3> input(20, 30, 50); - // ... set some values in input. - Tensor<float, 3> output(40, 90, 200); - output.stride({2, 3, 4}) = input; - +```cpp +Tensor<float, 3> input(20, 30, 50); +input.setRandom(); +Tensor<float, 3> output(40, 90, 200); +output.stride({2, 3, 4}) = input; +``` ### (Operation) slice(const StartIndices& offsets, const Sizes& extents) Returns a sub-tensor of the given tensor. For each dimension i, the slice is -made of the coefficients stored between offset[i] and offset[i] + extents[i] in +made of the coefficients stored between `offset[i]` and `offset[i] + extents[i]` in the input tensor. - Eigen::Tensor<int, 2> a(4, 3); - a.setValues({{0, 100, 200}, {300, 400, 500}, - {600, 700, 800}, {900, 1000, 1100}}); - Eigen::array<Eigen::Index, 2> offsets = {1, 0}; - Eigen::array<Eigen::Index, 2> extents = {2, 2}; - Eigen::Tensor<int, 2> slice = a.slice(offsets, extents); - cout << "a" << endl << a << endl; - => - a - 0 100 200 - 300 400 500 - 600 700 800 - 900 1000 1100 - cout << "slice" << endl << slice << endl; - => - slice - 300 400 - 600 700 +```cpp +Eigen::Tensor<int, 2> a(4, 3); +a.setValues({{0, 100, 200}, {300, 400, 500}, + {600, 700, 800}, {900, 1000, 1100}}); +Eigen::array<Eigen::Index, 2> offsets = {1, 0}; +Eigen::array<Eigen::Index, 2> extents = {2, 2}; +Eigen::Tensor<int, 2> slice = a.slice(offsets, extents); +std::cout << "a" << endl << a << endl; +// a +// 0 100 200 +// 300 400 500 +// 600 700 800 +// 900 1000 1100 + +std::cout << "slice" << endl << slice << endl; +// slice +// 300 400 +// 600 700 +``` + +### (Operation) stridedSlice(const StartIndices& start, const StopIndices& stop, const Strides& strides) + +Returns a sub-tensor by selecting elements using `start`, `stop` (exclusive), and `strides` for each dimension. + +This is similar to slicing in Python using [start:stop:step]. + +``` cpp +Eigen::Tensor<int, 2> a(4, 6); +a.setValues({{ 0, 10, 20, 30, 40, 50}, + {100, 110, 120, 130, 140, 150}, + {200, 210, 220, 230, 240, 250}, + {300, 310, 320, 330, 340, 350}}); + +Eigen::array<Eigen::Index, 2> start = {1, 1}; +Eigen::array<Eigen::Index, 2> stop = {4, 6}; // Stop is exclusive +Eigen::array<Eigen::Index, 2> strides = {2, 2}; + +Eigen::Tensor<int, 2> sub = a.stridedSlice(start, stop, strides); + +std::cout << "a\n" << a << "\n"; +std::cout << "sub\n" << sub << "\n"; + +// a +// 0 10 20 30 40 50 +// 100 110 120 130 140 150 +// 200 210 220 230 240 250 +// 300 310 320 330 340 350 + +// sub +// 110 130 150 +// 310 330 350 +``` +It is also possible to assign to a strided slice: + +``` cpp +Eigen::Tensor<int, 2> b(sub.dimensions()); +b.setConstant(-1); +a.stridedSlice(start, stop, strides) = b; +std::cout << "modified a\n" << a << "\n"; +// modified a +// 0 10 20 30 40 50 +// 100 -1 120 -1 140 -1 +// 200 210 220 230 240 250 +// 300 -1 320 -1 340 -1 + +``` ### (Operation) chip(const Index offset, const Index dim) -A chip is a special kind of slice. It is the subtensor at the given offset in -the dimension dim. The returned tensor has one fewer dimension than the input -tensor: the dimension dim is removed. +A chip is a special kind of slice. +It is the subtensor at the given offset in the dimension `dim`. -For example, a matrix chip would be either a row or a column of the input -matrix. +The returned tensor has one fewer dimension than the input tensor: the dimension dim is removed. - Eigen::Tensor<int, 2> a(4, 3); - a.setValues({{0, 100, 200}, {300, 400, 500}, - {600, 700, 800}, {900, 1000, 1100}}); - Eigen::Tensor<int, 1> row_3 = a.chip(2, 0); - Eigen::Tensor<int, 1> col_2 = a.chip(1, 1); - cout << "a" << endl << a << endl; - => - a - 0 100 200 - 300 400 500 - 600 700 800 - 900 1000 1100 - cout << "row_3" << endl << row_3 << endl; - => - row_3 - 600 700 800 - cout << "col_2" << endl << col_2 << endl; - => - col_2 - 100 400 700 1000 +For example, a matrix chip would be either a row or a column of the input matrix: + +```cpp +Eigen::Tensor<int, 2> a(4, 3); +a.setValues({{0, 100, 200}, {300, 400, 500}, + {600, 700, 800}, {900, 1000, 1100}}); +Eigen::Tensor<int, 1> row_3 = a.chip(2, 0); +Eigen::Tensor<int, 1> col_2 = a.chip(1, 1); +std::cout << "a\n" << a << "\n"; + +// a +// 0 100 200 +// 300 400 500 +// 600 700 800 +// 900 1000 1100 + +std::cout << "row_3\n" << row_3 << "\n"; +// row_3 +// 600 700 800 + +std::cout << "col_2\n" << col_2 << "\n"; +// col_2 +// 100 400 700 1000 +``` It is possible to assign values to a tensor chip since the chip operation is a lvalue. For example: - Eigen::Tensor<int, 1> a(3); - a.setValues({{100, 200, 300}}); - Eigen::Tensor<int, 2> b(2, 3); - b.setZero(); - b.chip(0, 0) = a; - cout << "a" << endl << a << endl; - => - a - 100 - 200 - 300 - cout << "b" << endl << b << endl; - => - b - 100 200 300 - 0 0 0 +```cpp +Eigen::Tensor<int, 1> a(3); +a.setValues({{100, 200, 300}}); +Eigen::Tensor<int, 2> b(2, 3); +b.setZero(); +b.chip(0, 0) = a; +std::cout << "a\n" << a << "\n"; +std::cout << "b\n" << b << "\n"; + +// a +// 100 +// 200 +// 300 + +// b +// 100 200 300 +// 0 0 0 +``` + + +The dimension can also be passed as a template parameter: + +```cpp +b.chip<0>(1) = a; // Equivalent to b.chip(1,0) = a; +``` + +Note that only one dimension can be chipped at a time. +To chip off multiple dimensions, you can chain calls + +```cpp +Eigen::Tensor<int, 3> a(2, 3, 4); +Eigen::Tensor<int, 1> b = b.chip<2>(0) // Now has shape [2,3] + .chip<1>(0); // Now has shape [2] +``` + +Be careful in which order you chip, as each operation affects the shape of the intermediate result. +For example: + +```cpp +// AVOID THIS +Eigen::Tensor<int, 1> c = b.chip<1>(0) // Now has shape [2,4] + .chip<1>(0); // Now has shape [2] +``` + +In general, it’s more intuitive to chip from the outermost dimension first. ### (Operation) reverse(const ReverseDimensions& reverse) @@ -1558,24 +1930,59 @@ For example this is what happens when you `reverse()` the first dimension of a 2D tensor: - Eigen::Tensor<int, 2> a(4, 3); - a.setValues({{0, 100, 200}, {300, 400, 500}, - {600, 700, 800}, {900, 1000, 1100}}); - Eigen::array<bool, 2> reverse({true, false}); - Eigen::Tensor<int, 2> b = a.reverse(reverse); - cout << "a" << endl << a << endl << "b" << endl << b << endl; - => - a - 0 100 200 - 300 400 500 - 600 700 800 - 900 1000 1100 - b - 900 1000 1100 - 600 700 800 - 300 400 500 - 0 100 200 +```cpp +Eigen::Tensor<int, 2> a(4, 3); +a.setValues({{0, 100, 200}, {300, 400, 500}, + {600, 700, 800}, {900, 1000, 1100}}); +Eigen::array<bool, 2> reverse({true, false}); +Eigen::Tensor<int, 2> b = a.reverse(reverse); +std::cout << "a\n" << a << "\n"; +std::cout << "b\n" << b << "\n"; +// a +// 0 100 200 +// 300 400 500 +// 600 700 800 +// 900 1000 1100 +// b +// 900 1000 1100 +// 600 700 800 +// 300 400 500 +// 0 100 200 +``` + +### (Operation) roll(const Rolls& shifts) + +Returns a tensor with the elements **circularly shifted** (like bit rotation) along one or more dimensions. + +For each dimension `i`, the content is shifted by `shifts[i]` positions: + +- A **positive shift** of `+s` moves each value to a **lower index** by `s`. +- A **negative shift** of `-s` moves each value to a **higher index** by `s`. + +```cpp +Eigen::Tensor<int, 2> a(3, 4); +a.setValues({{ 1, 2, 3, 4}, + { 5, 6, 7, 8}, + { 9, 10, 11, 12}}); + +Eigen::array<Eigen::Index, 2> shifts = {1, -2}; + +Eigen::Tensor<int, 2> rolled = a.roll(shifts); + +std::cout << "a\n" << a << "\n"; +std::cout << "rolled\n" << rolled << "\n"; + +// a +// 1 2 3 4 +// 5 6 7 8 +// 9 10 11 12 +// +// rolled +// 7 8 5 6 +// 11 12 9 10 +// 3 4 1 2 +``` ### (Operation) broadcast(const Broadcast& broadcast) @@ -1584,97 +1991,152 @@ The broadcast argument specifies how many copies of the input tensor need to be made in each of the dimensions. - Eigen::Tensor<int, 2> a(2, 3); - a.setValues({{0, 100, 200}, {300, 400, 500}}); - Eigen::array<int, 2> bcast({3, 2}); - Eigen::Tensor<int, 2> b = a.broadcast(bcast); - cout << "a" << endl << a << endl << "b" << endl << b << endl; - => - a - 0 100 200 - 300 400 500 - b - 0 100 200 0 100 200 - 300 400 500 300 400 500 - 0 100 200 0 100 200 - 300 400 500 300 400 500 - 0 100 200 0 100 200 - 300 400 500 300 400 500 +```cpp +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{0, 100, 200}, {300, 400, 500}}); +Eigen::array<int, 2> bcast({3, 2}); +Eigen::Tensor<int, 2> b = a.broadcast(bcast); +std::cout << "a" << endl << a << endl << "b" << endl << b << endl; +// a +// 0 100 200 +// 300 400 500 +// b +// 0 100 200 0 100 200 +// 300 400 500 300 400 500 +// 0 100 200 0 100 200 +// 300 400 500 300 400 500 +// 0 100 200 0 100 200 +// 300 400 500 300 400 500 +``` + +Note: Broadcasting does not increase rank. +To broadcast into higher dimensions, you must first reshape the tensor with singleton (1) dimensions: + +```cpp +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{0, 100, 200}, {300, 400, 500}}); + +Eigen::array<Eigen::Index, 3> new_shape = {1, 2, 3}; //Reshape to [1, 2, 3] +Eigen::array<int, 3> bcast = {4, 1, 1}; // Broadcast to [4, 2, 3] +Eigen::Tensor<int, 3> b = a.reshape(new_shape).broadcast(bcast); + +std::cout << "b dimensions: " << b.dimensions() << "\n"; +std::cout << b << "\n"; +``` ### (Operation) concatenate(const OtherDerived& other, Axis axis) -TODO +Returns a view of two tensors joined along a specified axis. +The dimensions of the two tensors must match on all axes except the concatenation axis. +The resulting tensor has the same rank as the inputs. + +```cpp +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{0, 100, 200}, {300, 400, 500}}); + +Eigen::Tensor<int, 2> b(2, 3); +b.setValues({{-1, -2, -3}, {-4, -5, -6}}); + +// Concatenate along dimension 0: resulting shape is [4, 3] +Eigen::Tensor<int, 2> c = a.concatenate(b, 0); + +// Concatenate along dimension 1: resulting shape is [2, 6] +Eigen::Tensor<int, 2> d = a.concatenate(b, 1); + +std::cout << "a\n" << a << "\n" + << "b\n" << b << "\n" + << "c (concatenated along dim 0)\n" << c << "\n" + << "d (concatenated along dim 1)\n" << d << "\n"; +// a +// 0 100 200 +// 300 400 500 +// b +// -1 -2 -3 +// -4 -5 -6 +// c (concatenated along dim 0) +// 0 100 200 +// 300 400 500 +// -1 -2 -3 +// -4 -5 -6 +// d (concatenated along dim 1) +// 0 100 200 -1 -2 -3 +// 300 400 500 -4 -5 -6 +``` ### (Operation) pad(const PaddingDimensions& padding) Returns a view of the input tensor in which the input is padded with zeros. - Eigen::Tensor<int, 2> a(2, 3); - a.setValues({{0, 100, 200}, {300, 400, 500}}); - Eigen::array<pair<int, int>, 2> paddings; - paddings[0] = make_pair(0, 1); - paddings[1] = make_pair(2, 3); - Eigen::Tensor<int, 2> b = a.pad(paddings); - cout << "a" << endl << a << endl << "b" << endl << b << endl; - => - a - 0 100 200 - 300 400 500 - b - 0 0 0 0 - 0 0 0 0 - 0 100 200 0 - 300 400 500 0 - 0 0 0 0 - 0 0 0 0 - 0 0 0 0 - +```cpp +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{0, 100, 200}, {300, 400, 500}}); +Eigen::array<pair<int, int>, 2> paddings; +paddings[0] = make_pair(0, 1); +paddings[1] = make_pair(2, 3); +Eigen::Tensor<int, 2> b = a.pad(paddings); +std::cout << "a" << endl << a << endl << "b" << endl << b << endl; +// a +// 0 100 200 +// 300 400 500 +// b +// 0 0 0 0 +// 0 0 0 0 +// 0 100 200 0 +// 300 400 500 0 +// 0 0 0 0 +// 0 0 0 0 +// 0 0 0 0 +``` ### (Operation) extract_patches(const PatchDims& patch_dims) Returns a tensor of coefficient patches extracted from the input tensor, where -each patch is of dimension specified by 'patch_dims'. The returned tensor has +each patch is of dimension specified by `patch_dims`. The returned tensor has one greater dimension than the input tensor, which is used to index each patch. The patch index in the output tensor depends on the data layout of the input -tensor: the patch index is the last dimension ColMajor layout, and the first -dimension in RowMajor layout. +tensor: the patch index is the last dimension `ColMajor` layout, and the first +dimension in `RowMajor` layout. For example, given the following input tensor: - Eigen::Tensor<float, 2, DataLayout> tensor(3,4); - tensor.setValues({{0.0f, 1.0f, 2.0f, 3.0f}, - {4.0f, 5.0f, 6.0f, 7.0f}, - {8.0f, 9.0f, 10.0f, 11.0f}}); +```cpp +Eigen::Tensor<float, 2, DataLayout> tensor(3,4); +tensor.setValues({{0.0f, 1.0f, 2.0f, 3.0f}, + {4.0f, 5.0f, 6.0f, 7.0f}, + {8.0f, 9.0f, 10.0f, 11.0f}}); - cout << "tensor: " << endl << tensor << endl; - => - tensor: - 0 1 2 3 - 4 5 6 7 - 8 9 10 11 +std::cout << "tensor: " << endl << tensor << endl; + +// tensor: +// 0 1 2 3 +// 4 5 6 7 +// 8 9 10 11 +``` Six 2x2 patches can be extracted and indexed using the following code: - Eigen::Tensor<float, 3, DataLayout> patch; - Eigen::array<ptrdiff_t, 2> patch_dims; - patch_dims[0] = 2; - patch_dims[1] = 2; - patch = tensor.extract_patches(patch_dims); - for (int k = 0; k < 6; ++k) { - cout << "patch index: " << k << endl; - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 2; ++j) { - if (DataLayout == ColMajor) { - cout << patch(i, j, k) << " "; - } else { - cout << patch(k, i, j) << " "; - } - } - cout << endl; +```cpp +Eigen::Tensor<float, 3, DataLayout> patch; +Eigen::array<ptrdiff_t, 2> patch_dims; +patch_dims[0] = 2; +patch_dims[1] = 2; +patch = tensor.extract_patches(patch_dims); +for (int k = 0; k < 6; ++k) { + std::cout << "patch index: " << k << endl; + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 2; ++j) { + if (DataLayout == ColMajor) { + std::cout << patch(i, j, k) << " "; + } else { + std::cout << patch(k, i, j) << " "; } } + std::cout << endl; + } +} +``` -This code results in the following output when the data layout is ColMajor: +This code results in the following output when the data layout is `ColMajor`: patch index: 0 0 1 @@ -1696,7 +2158,8 @@ 10 11 This code results in the following output when the data layout is RowMajor: -(NOTE: the set of patches is the same as in ColMajor, but are indexed differently). + +**NOTE**: the set of patches is the same as in `ColMajor`, but are indexed differently patch index: 0 0 1 @@ -1723,119 +2186,141 @@ which is expected to have dimensions ordered as follows (depending on the data layout of the input tensor, and the number of additional dimensions 'N'): -*) ColMajor -1st dimension: channels (of size d) -2nd dimension: rows (of size r) -3rd dimension: columns (of size c) -4th-Nth dimension: time (for video) or batch (for bulk processing). +- `ColMajor` + - 1st dimension: channels (of size d) + - 2nd dimension: rows (of size r) + - 3rd dimension: columns (of size c) + - 4th-Nth dimension: time (for video) or batch (for bulk processing). -*) RowMajor (reverse order of ColMajor) -1st-Nth dimension: time (for video) or batch (for bulk processing). -N+1'th dimension: columns (of size c) -N+2'th dimension: rows (of size r) -N+3'th dimension: channels (of size d) +* `RowMajor` (reverse order of `ColMajor`) + - 1st-Nth dimension: time (for video) or batch (for bulk processing). + - N+1'th dimension: columns (of size c) + - N+2'th dimension: rows (of size r) + - N+3'th dimension: channels (of size d) The returned tensor has one greater dimension than the input tensor, which is used to index each patch. The patch index in the output tensor depends on the data layout of the input tensor: the patch index is the 4'th dimension in -ColMajor layout, and the 4'th from the last dimension in RowMajor layout. +`ColMajor` layout, and the 4'th from the last dimension in `RowMajor` layout. For example, given the following input tensor with the following dimension sizes: - *) depth: 2 - *) rows: 3 - *) columns: 5 - *) batch: 7 +- depth: 2 +- rows: 3 +- columns: 5 +- batch: 7 - Tensor<float, 4> tensor(2,3,5,7); - Tensor<float, 4, RowMajor> tensor_row_major = tensor.swap_layout(); +```cpp +Tensor<float, 4> tensor(2,3,5,7); +Tensor<float, 4, RowMajor> tensor_row_major = tensor.swap_layout(); +``` 2x2 image patches can be extracted and indexed using the following code: -*) 2D patch: ColMajor (patch indexed by second-to-last dimension) +#### 2D patch: `ColMajor` (patch indexed by second-to-last dimension) - Tensor<float, 5> twod_patch; - twod_patch = tensor.extract_image_patches<2, 2>(); - // twod_patch.dimension(0) == 2 - // twod_patch.dimension(1) == 2 - // twod_patch.dimension(2) == 2 - // twod_patch.dimension(3) == 3*5 - // twod_patch.dimension(4) == 7 +```cpp +Tensor<float, 5> twod_patch; +twod_patch = tensor.extract_image_patches<2, 2>(); +// twod_patch.dimension(0) == 2 +// twod_patch.dimension(1) == 2 +// twod_patch.dimension(2) == 2 +// twod_patch.dimension(3) == 3*5 +// twod_patch.dimension(4) == 7 +``` -*) 2D patch: RowMajor (patch indexed by the second dimension) +#### 2D patch: `RowMajor` (patch indexed by the second dimension) - Tensor<float, 5, RowMajor> twod_patch_row_major; - twod_patch_row_major = tensor_row_major.extract_image_patches<2, 2>(); - // twod_patch_row_major.dimension(0) == 7 - // twod_patch_row_major.dimension(1) == 3*5 - // twod_patch_row_major.dimension(2) == 2 - // twod_patch_row_major.dimension(3) == 2 - // twod_patch_row_major.dimension(4) == 2 +```cpp +Tensor<float, 5, RowMajor> twod_patch_row_major; +twod_patch_row_major = tensor_row_major.extract_image_patches<2, 2>(); +// twod_patch_row_major.dimension(0) == 7 +// twod_patch_row_major.dimension(1) == 3*5 +// twod_patch_row_major.dimension(2) == 2 +// twod_patch_row_major.dimension(3) == 2 +// twod_patch_row_major.dimension(4) == 2 +``` ## Special Operations ### (Operation) cast<T>() -Returns a tensor of type T with the same dimensions as the original tensor. +Returns a tensor of type `T` with the same dimensions as the original tensor. The returned tensor contains the values of the original tensor converted to -type T. +type `T`. - Eigen::Tensor<float, 2> a(2, 3); - Eigen::Tensor<int, 2> b = a.cast<int>(); +```cpp +Eigen::Tensor<float, 2> a(2, 3); +Eigen::Tensor<int, 2> b = a.cast<int>(); +``` This can be useful for example if you need to do element-wise division of -Tensors of integers. This is not currently supported by the Tensor library +Tensors of integers. +This is not currently supported by the Tensor library but you can easily cast the tensors to floats to do the division: - Eigen::Tensor<int, 2> a(2, 3); - a.setValues({{0, 1, 2}, {3, 4, 5}}); - Eigen::Tensor<int, 2> b = - (a.cast<float>() / a.constant(2).cast<float>()).cast<int>(); - cout << "a" << endl << a << endl << endl; - cout << "b" << endl << b << endl << endl; - => - a - 0 1 2 - 3 4 5 +```cpp +Eigen::Tensor<int, 2> a(2, 3); +a.setValues({{0, 1, 2}, {3, 4, 5}}); +Eigen::Tensor<int, 2> b = + (a.cast<float>() / a.constant(2).cast<float>()).cast<int>(); +std::cout << "a\n" << a << "\n"; +std::cout << "b\n" << b << "\n"; - b - 0 0 1 - 1 2 2 - +// a +// 0 1 2 +// 3 4 5 +// +// b +// 0 0 1 +// 1 2 2 +``` ### (Operation) eval() +See **Calling eval()**. -TODO + ## Tensor Printing Tensors can be printed into a stream object (e.g. `std::cout`) using different formatting options. - Eigen::Tensor<float, 3> tensor3d = {4, 3, 2}; - tensor3d.setValues( {{{1, 2}, {3, 4}, {5, 6}}, {{7, 8}, {9, 10}, {11, 12}}, {{13, 14}, {15, 16}, {17, 18}}, {{19, 20}, {21, 22}, {23, 24}}} ); - std::cout << tensor3d.format(Eigen::TensorIOFormat::Plain()) << std::endl; - ==> - 1 2 - 3 4 - 5 6 - - 7 8 - 9 10 - 11 12 - - 13 14 - 15 16 - 17 18 - - 19 20 - 21 22 - 23 24 - +```cpp +Eigen::Tensor<float, 3> tensor3d = {4, 3, 2}; +tensor3d.setValues( {{{1, 2}, + {3, 4}, + {5, 6}}, + {{7, 8}, + {9, 10}, + {11, 12}}, + {{13, 14}, + {15, 16}, + {17, 18}}, + {{19, 20}, + {21, 22}, + {23, 24}}} ); +std::cout << tensor3d.format(Eigen::TensorIOFormat::Plain()) << ; +// 1 2 +// 3 4 +// 5 6 +// +// 7 8 +// 9 10 +// 11 12 +// +// 13 14 +// 15 16 +// 17 18 +// +// 19 20 +// 21 22 +// 23 24 +``` In the example, we used the predefined format `Eigen::TensorIOFormat::Plain`. Here is the list of all predefined formats from which you can choose: - `Eigen::TensorIOFormat::Plain()` for a plain output without braces. Different submatrices are separated by a blank line. - `Eigen::TensorIOFormat::Numpy()` for numpy-like output. -- `Eigen::TensorIOFormat::Native()` for a `c++` like output which can be directly copy-pasted to setValues(). +- `Eigen::TensorIOFormat::Native()` for a `c++` like output which can be directly copy-pasted to `setValues()`. - `Eigen::TensorIOFormat::Legacy()` for a backwards compatible printing of tensors. If you send the tensor directly to the stream the default format is called which is `Eigen::IOFormats::Plain()`. @@ -1849,14 +2334,19 @@ ## Representation of scalar values -Scalar values are often represented by tensors of size 1 and rank 0.For example -Tensor<T, N>::maximum() currently returns a Tensor<T, 0>. Similarly, the inner -product of 2 1d tensors (through contractions) returns a 0d tensor. +Scalar values are often represented by tensors of size 1 and rank 0. + +For example `Tensor<T, N>::maximum()` returns a `Tensor<T, 0>`. + +Similarly, the inner product of 2 1d tensors (through contractions) returns a 0d tensor. + +The scalar value can be extracted as explained in **Reduction along all dimensions**. + ## Limitations * The number of tensor dimensions is currently limited to 250 when using a compiler that supports cxx11. It is limited to only 5 for older compilers. -* The IndexList class requires a cxx11 compliant compiler. You can use an +* The `IndexList` class requires a cxx11 compliant compiler. You can use an array of indices instead if you don't have access to a modern compiler. * On GPUs only floating point values are properly tested and optimized for.
diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceGpu.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceGpu.h index 4c24bc1..2a3b087 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceGpu.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceGpu.h
@@ -10,14 +10,11 @@ #if defined(EIGEN_USE_GPU) && !defined(EIGEN_CXX11_TENSOR_TENSOR_DEVICE_GPU_H) #define EIGEN_CXX11_TENSOR_TENSOR_DEVICE_GPU_H -// This header file container defines fo gpu* macros which will resolve to -// their equivalent hip* or cuda* versions depending on the compiler in use -// A separate header (included at the end of this file) will undefine all -#include "TensorGpuHipCudaDefines.h" - // IWYU pragma: private #include "./InternalHeaderCheck.h" +#include "../../../../../Eigen/src/Core/util/GpuHipCudaDefines.inc" + namespace Eigen { static const int kGpuScratchSize = 1024; @@ -390,6 +387,6 @@ } // end namespace Eigen // undefine all the gpu* macros we defined at the beginning of the file -#include "TensorGpuHipCudaUndefines.h" +#include "../../../../../Eigen/src/Core/util/GpuHipCudaUndefines.inc" #endif // EIGEN_CXX11_TENSOR_TENSOR_DEVICE_GPU_H
diff --git a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h index 1c3ce87..387836b 100644 --- a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h +++ b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h
@@ -323,7 +323,7 @@ const T num = ppolevl<T, 3>::run(q2, gamma); const T denom = pmul(x, ppolevl<T, 4>::run(q2, delta)); const T r = pdiv(num, denom); - const T maybe_two = pand(pcmp_lt(x, pset1<T>(0.0)), pset1<T>(2.0)); + const T maybe_two = pselect(pcmp_lt(x, pset1<T>(0.0f)), pset1<T>(2.0f), pset1<T>(0.0f)); const T erfc_large = pmadd(z, r, maybe_two); return pselect(x_abs_gt_one_mask, erfc_large, erfc_small); } @@ -397,7 +397,7 @@ const T num_large = ppolevl<T, 9>::run(q2, gamma); const T denom_large = pmul(x, ppolevl<T, 9>::run(q2, delta)); const T r = pdiv(num_large, denom_large); - const T maybe_two = pand(pcmp_lt(x, pset1<T>(0.0)), pset1<T>(2.0)); + const T maybe_two = pselect(pcmp_lt(x, pset1<T>(0.0)), pset1<T>(2.0), pset1<T>(0.0)); return pmadd(z, r, maybe_two); }
diff --git a/unsupported/test/cxx11_tensor_argmax_gpu.cu b/unsupported/test/cxx11_tensor_argmax_gpu.cu index 629e5c0..90c94c5 100644 --- a/unsupported/test/cxx11_tensor_argmax_gpu.cu +++ b/unsupported/test/cxx11_tensor_argmax_gpu.cu
@@ -14,8 +14,6 @@ #include "main.h" #include <unsupported/Eigen/CXX11/Tensor> -#include <unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> - using Eigen::Tensor; template <int Layout>
diff --git a/unsupported/test/cxx11_tensor_contract_gpu.cu b/unsupported/test/cxx11_tensor_contract_gpu.cu index c9eebfc..a41a2df 100644 --- a/unsupported/test/cxx11_tensor_contract_gpu.cu +++ b/unsupported/test/cxx11_tensor_contract_gpu.cu
@@ -17,8 +17,6 @@ #include "main.h" #include <unsupported/Eigen/CXX11/Tensor> -#include <unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> - using Eigen::Tensor; typedef Tensor<float, 1>::DimensionPair DimPair;
diff --git a/unsupported/test/cxx11_tensor_device.cu b/unsupported/test/cxx11_tensor_device.cu index 0a26ab9..b4ba23e 100644 --- a/unsupported/test/cxx11_tensor_device.cu +++ b/unsupported/test/cxx11_tensor_device.cu
@@ -17,8 +17,6 @@ #include "OffByOneScalar.h" #include <unsupported/Eigen/CXX11/Tensor> -#include <unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> - using Eigen::RowMajor; using Eigen::Tensor;
diff --git a/unsupported/test/cxx11_tensor_gpu.cu b/unsupported/test/cxx11_tensor_gpu.cu index c5046c0..c4d0be1 100644 --- a/unsupported/test/cxx11_tensor_gpu.cu +++ b/unsupported/test/cxx11_tensor_gpu.cu
@@ -15,8 +15,6 @@ #include "main.h" #include <unsupported/Eigen/CXX11/Tensor> -#include <unsupported/Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> - using Eigen::Tensor; void test_gpu_nullary() {
diff --git a/unsupported/test/cxx11_tensor_random_gpu.cu b/unsupported/test/cxx11_tensor_random_gpu.cu index fa8da7d..6edea1c 100644 --- a/unsupported/test/cxx11_tensor_random_gpu.cu +++ b/unsupported/test/cxx11_tensor_random_gpu.cu
@@ -16,8 +16,6 @@ #include "main.h" #include <Eigen/CXX11/Tensor> -#include <Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> - void test_gpu_random_uniform() { Tensor<float, 2> out(72, 97); out.setZero();
diff --git a/unsupported/test/cxx11_tensor_scan_gpu.cu b/unsupported/test/cxx11_tensor_scan_gpu.cu index c9997ad..b94ab31 100644 --- a/unsupported/test/cxx11_tensor_scan_gpu.cu +++ b/unsupported/test/cxx11_tensor_scan_gpu.cu
@@ -16,8 +16,6 @@ #include "main.h" #include <unsupported/Eigen/CXX11/Tensor> -#include <Eigen/CXX11/src/Tensor/TensorGpuHipCudaDefines.h> - using Eigen::Tensor; typedef Tensor<float, 1>::DimensionPair DimPair;