No public description PiperOrigin-RevId: 681078766 Change-Id: I2d62f47fc9796c0785b23c6a0933501f58c273bc
diff --git a/Eigen/src/Core/CoreEvaluators.h b/Eigen/src/Core/CoreEvaluators.h index 3d78fd8..879b0db 100644 --- a/Eigen/src/Core/CoreEvaluators.h +++ b/Eigen/src/Core/CoreEvaluators.h
@@ -217,7 +217,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return const_cast<Scalar*>(m_d.data)[index]; } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { if (IsRowMajor) return ploadt<PacketType, LoadMode>(m_d.data + row * m_d.outerStride() + col); else @@ -225,12 +225,12 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return ploadt<PacketType, LoadMode>(m_d.data + index); } template <int StoreMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { if (IsRowMajor) return pstoret<Scalar, PacketType, StoreMode>(const_cast<Scalar*>(m_d.data) + row * m_d.outerStride() + col, x); else @@ -238,7 +238,7 @@ } template <int StoreMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { return pstoret<Scalar, PacketType, StoreMode>(const_cast<Scalar*>(m_d.data) + index, x); } @@ -296,22 +296,22 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_argImpl.template packet<LoadMode, PacketType>(col, row); } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_argImpl.template packet<LoadMode, PacketType>(index); } template <int StoreMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { m_argImpl.template writePacket<StoreMode, PacketType>(col, row, x); } template <int StoreMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { m_argImpl.template writePacket<StoreMode, PacketType>(index, x); } @@ -490,12 +490,12 @@ } template <int LoadMode, typename PacketType, typename IndexType> - EIGEN_STRONG_INLINE PacketType packet(IndexType row, IndexType col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(IndexType row, IndexType col) const { return m_wrapper.template packetOp<PacketType>(m_functor, row, col); } template <int LoadMode, typename PacketType, typename IndexType> - EIGEN_STRONG_INLINE PacketType packet(IndexType index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(IndexType index) const { return m_wrapper.template packetOp<PacketType>(m_functor, index); } @@ -534,12 +534,12 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_d.func().packetOp(m_d.argImpl.template packet<LoadMode, PacketType>(row, col)); } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_d.func().packetOp(m_d.argImpl.template packet<LoadMode, PacketType>(index)); } @@ -627,7 +627,7 @@ } template <int LoadMode, typename PacketType = SrcPacketType> - EIGEN_STRONG_INLINE PacketType srcPacket(Index row, Index col, Index offset) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType srcPacket(Index row, Index col, Index offset) const { constexpr int PacketSize = unpacket_traits<PacketType>::size; Index actualRow = IsRowMajor ? row : row + (offset * PacketSize); Index actualCol = IsRowMajor ? col + (offset * PacketSize) : col; @@ -635,7 +635,7 @@ return m_argImpl.template packet<LoadMode, PacketType>(actualRow, actualCol); } template <int LoadMode, typename PacketType = SrcPacketType> - EIGEN_STRONG_INLINE PacketType srcPacket(Index index, Index offset) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType srcPacket(Index index, Index offset) const { constexpr int PacketSize = unpacket_traits<PacketType>::size; Index actualIndex = index + (offset * PacketSize); eigen_assert(check_array_bounds(actualIndex, PacketSize) && "Array index out of bounds"); @@ -652,7 +652,7 @@ // If not, perform full load. Otherwise, revert to a scalar loop to perform a partial load. // In either case, perform a vectorized cast of the source packet. template <int LoadMode, typename DstPacketType, AltSrcScalarOp<DstPacketType> = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { constexpr int DstPacketSize = unpacket_traits<DstPacketType>::size; constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode); @@ -669,7 +669,7 @@ } // Use the source packet type with the same size as DstPacketType, if it exists template <int LoadMode, typename DstPacketType, SrcPacketArgs1<DstPacketType> = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { constexpr int DstPacketSize = unpacket_traits<DstPacketType>::size; using SizedSrcPacketType = typename find_packet_by_size<SrcType, DstPacketSize>::type; constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); @@ -678,14 +678,14 @@ } // unpacket_traits<DstPacketType>::size == 2 * SrcPacketSize template <int LoadMode, typename DstPacketType, SrcPacketArgs2<DstPacketType> = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast<SrcPacketType, DstPacketType>(srcPacket<SrcLoadMode>(row, col, 0), srcPacket<SrcLoadMode>(row, col, 1)); } // unpacket_traits<DstPacketType>::size == 4 * SrcPacketSize template <int LoadMode, typename DstPacketType, SrcPacketArgs4<DstPacketType> = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast<SrcPacketType, DstPacketType>(srcPacket<SrcLoadMode>(row, col, 0), srcPacket<SrcLoadMode>(row, col, 1), srcPacket<SrcLoadMode>(row, col, 2), @@ -693,7 +693,7 @@ } // unpacket_traits<DstPacketType>::size == 8 * SrcPacketSize template <int LoadMode, typename DstPacketType, SrcPacketArgs8<DstPacketType> = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index row, Index col) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast<SrcPacketType, DstPacketType>( srcPacket<SrcLoadMode>(row, col, 0), srcPacket<SrcLoadMode>(row, col, 1), srcPacket<SrcLoadMode>(row, col, 2), @@ -703,7 +703,7 @@ // Analogous routines for linear access. template <int LoadMode, typename DstPacketType, AltSrcScalarOp<DstPacketType> = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { constexpr int DstPacketSize = unpacket_traits<DstPacketType>::size; constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); constexpr int SrcLoadMode = plain_enum_min(SrcBytesIncrement, LoadMode); @@ -719,7 +719,7 @@ return pcast<SrcPacketType, DstPacketType>(src); } template <int LoadMode, typename DstPacketType, SrcPacketArgs1<DstPacketType> = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { constexpr int DstPacketSize = unpacket_traits<DstPacketType>::size; using SizedSrcPacketType = typename find_packet_by_size<SrcType, DstPacketSize>::type; constexpr int SrcBytesIncrement = DstPacketSize * sizeof(SrcType); @@ -727,18 +727,18 @@ return pcast<SizedSrcPacketType, DstPacketType>(srcPacket<SrcLoadMode, SizedSrcPacketType>(index, 0)); } template <int LoadMode, typename DstPacketType, SrcPacketArgs2<DstPacketType> = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast<SrcPacketType, DstPacketType>(srcPacket<SrcLoadMode>(index, 0), srcPacket<SrcLoadMode>(index, 1)); } template <int LoadMode, typename DstPacketType, SrcPacketArgs4<DstPacketType> = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast<SrcPacketType, DstPacketType>(srcPacket<SrcLoadMode>(index, 0), srcPacket<SrcLoadMode>(index, 1), srcPacket<SrcLoadMode>(index, 2), srcPacket<SrcLoadMode>(index, 3)); } template <int LoadMode, typename DstPacketType, SrcPacketArgs8<DstPacketType> = true> - EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DstPacketType packet(Index index) const { constexpr int SrcLoadMode = plain_enum_min(SrcPacketBytes, LoadMode); return pcast<SrcPacketType, DstPacketType>(srcPacket<SrcLoadMode>(index, 0), srcPacket<SrcLoadMode>(index, 1), srcPacket<SrcLoadMode>(index, 2), srcPacket<SrcLoadMode>(index, 3), @@ -810,14 +810,14 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_d.func().packetOp(m_d.arg1Impl.template packet<LoadMode, PacketType>(row, col), m_d.arg2Impl.template packet<LoadMode, PacketType>(row, col), m_d.arg3Impl.template packet<LoadMode, PacketType>(row, col)); } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_d.func().packetOp(m_d.arg1Impl.template packet<LoadMode, PacketType>(index), m_d.arg2Impl.template packet<LoadMode, PacketType>(index), m_d.arg3Impl.template packet<LoadMode, PacketType>(index)); @@ -908,13 +908,13 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_d.func().packetOp(m_d.lhsImpl.template packet<LoadMode, PacketType>(row, col), m_d.rhsImpl.template packet<LoadMode, PacketType>(row, col)); } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_d.func().packetOp(m_d.lhsImpl.template packet<LoadMode, PacketType>(index), m_d.rhsImpl.template packet<LoadMode, PacketType>(index)); } @@ -1030,24 +1030,24 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_data[index * m_innerStride.value()]; } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { PointerType ptr = m_data + row * rowStride() + col * colStride(); return internal::ploadt<PacketType, LoadMode>(ptr); } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return internal::ploadt<PacketType, LoadMode>(m_data + index * m_innerStride.value()); } template <int StoreMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { PointerType ptr = m_data + row * rowStride() + col * colStride(); return internal::pstoret<Scalar, PacketType, StoreMode>(ptr, x); } template <int StoreMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { internal::pstoret<Scalar, PacketType, StoreMode>(m_data + index * m_innerStride.value(), x); } @@ -1217,12 +1217,12 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_argImpl.template packet<LoadMode, PacketType>(m_startRow.value() + row, m_startCol.value() + col); } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { if (ForwardLinearAccess) return m_argImpl.template packet<LoadMode, PacketType>(m_linear_offset.value() + index); else @@ -1230,12 +1230,12 @@ } template <int StoreMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { return m_argImpl.template writePacket<StoreMode, PacketType>(m_startRow.value() + row, m_startCol.value() + col, x); } template <int StoreMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { if (ForwardLinearAccess) return m_argImpl.template writePacket<StoreMode, PacketType>(m_linear_offset.value() + index, x); else @@ -1378,7 +1378,7 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { const Index actual_row = internal::traits<XprType>::RowsAtCompileTime == 1 ? 0 : RowFactor == 1 ? row : row % m_rows.value(); @@ -1390,7 +1390,7 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { const Index actual_index = internal::traits<XprType>::RowsAtCompileTime == 1 ? (ColFactor == 1 ? index : index % m_cols.value()) : (RowFactor == 1 ? index : index % m_rows.value()); @@ -1435,22 +1435,22 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { return m_argImpl.coeffRef(index); } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { return m_argImpl.template packet<LoadMode, PacketType>(row, col); } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_argImpl.template packet<LoadMode, PacketType>(index); } template <int StoreMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { m_argImpl.template writePacket<StoreMode>(row, col, x); } template <int StoreMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { m_argImpl.template writePacket<StoreMode>(index, x); } @@ -1532,7 +1532,7 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { enum { PacketSize = unpacket_traits<PacketType>::size, OffsetRow = ReverseRow && IsColMajor ? PacketSize : 1, @@ -1544,14 +1544,14 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { enum { PacketSize = unpacket_traits<PacketType>::size }; return preverse( m_argImpl.template packet<LoadMode, PacketType>(m_rows.value() * m_cols.value() - index - PacketSize)); } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index row, Index col, const PacketType& x) { // FIXME we could factorize some code with packet(i,j) enum { PacketSize = unpacket_traits<PacketType>::size, @@ -1565,7 +1565,7 @@ } template <int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketType& x) { enum { PacketSize = unpacket_traits<PacketType>::size }; m_argImpl.template writePacket<LoadMode>(m_rows.value() * m_cols.value() - index - PacketSize, preverse(x)); }
diff --git a/Eigen/src/Core/InnerProduct.h b/Eigen/src/Core/InnerProduct.h index 38689da..1e16942 100644 --- a/Eigen/src/Core/InnerProduct.h +++ b/Eigen/src/Core/InnerProduct.h
@@ -85,13 +85,13 @@ } template <typename PacketType, int LhsMode = LhsAlignment, int RhsMode = RhsAlignment> - EIGEN_STRONG_INLINE PacketType packet(Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { return m_func.packet(m_lhs.template packet<LhsMode, PacketType>(index), m_rhs.template packet<RhsMode, PacketType>(index)); } template <typename PacketType, int LhsMode = LhsAlignment, int RhsMode = RhsAlignment> - EIGEN_STRONG_INLINE PacketType packet(const PacketType& value, Index index) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(const PacketType& value, Index index) const { return m_func.packet(value, m_lhs.template packet<LhsMode, PacketType>(index), m_rhs.template packet<RhsMode, PacketType>(index)); }