Update Eigen to commit:b1f8b0c3b8db9c3711349144acf951226700ddb4 CHANGELOG ========= b1f8b0c3b - Core: Support flexible indexers in `VectorwiseOp::replicate` 8f1a18ff7 - Core: Gate product_packet_cascade on MightVectorize and mark assignPacket device-callable 630f81dbf - RVV: Enable generic packet sinh, cosh, asinh, acosh, atan, atanh, log10, log1p, expm1, pow, cbrt, rsqrt and erfc 5b5b44a71 - LU: Unroll PartialPivLU for fixed sizes up to 12 32a4f127f - Eigenvalues: Defer far-column reflector updates in the Francis QR step ab8a55665 - Householder: Apply two- and three-element reflectors from the right in one pass 97898f371 - AltiVec: Fix out-of-bounds tail accesses in VSX and MMA GEMM kernels 0013bcb18 - Geometry: Preserve single-rounding pivot and trace order in Quaternion(Matrix3) 07d4528c8 - MSVC: Fix the ILP64 BLAS DLL and the stencil test 2774a036e - StructuredMatrices: Identity and nested factors for KroneckerOperator 869eb9850 - Core: Pin the TRSM padded tail's view of the right-hand side to column-major b6225b7dc - Core: Finish partial GEMV packets in one pass with masked segments c541f8f43 - Core: Support flexible indexers in `DenseBase::replicate` 7f03d8469 - NEON: Classify floats with absolute comparisons and reduce allFinite() through masks bc99fddc6 - Cholesky: Unblocked LLT up to 48 columns and 16-column minimum blocks e3755c9c7 - Core: Scalar tails for assignments bounded to a few packets ff16ae80a - Added specific value tests (x2) and random number generated tests for autodiff... 03d21656a - StructuredMatrices: Reuse Cauchy reciprocal columns across right-hand sides 477435767 - SME: Fold a real alpha into the complex tile-pair store c8725af72 - StructuredMatrices: Drop extreme-scale hardening from DPR1EigenSolver 3346f1a2b - StructuredMatrices: Drop extreme-scale hardening from DiagonalPlusLowRank a29e46e2e - StructuredMatrices: Solve Kronecker least squares by per-factor COD 036a81d7c - NEON: Clean up complex arithmetic and fix GCC ARM32 float compares 1c7d00c27 - Core: Fix out-of-bounds B-panel lookahead loads in AVX512 GEMM kernel 4be755d45 - Core: Decouple the fixed-size product threshold and retune it for SME dee1af8bb - Householder: Restore AutoDiffScalar support in QR and JacobiSVD f7a2dd7a9 - FindCoeff: Require HasCmp for the packet path of minCoeff/maxCoeff a82afb7c7 - Geometry: Fix AngleAxis::fromRotationMatrix accuracy and axis sign near pi 7194105cc - StructuredMatrices: Drop extreme-scale hardening from the FFT operators d11bb9121 - NEON: Optimize stride-two float gathers 7ac31bc91 - EulerAngles: Test the constructor from a data pointer 038961d50 - NumericalDiff: Implement Fornberg finite difference stencil algorithm b4befd43f - SVD: add PreconditionSquareMatrix option. 982e195f9 - StructuredMatrices: Batch KroneckerOperator products and solves, drop extreme-scale hardening 221aa496a - StructuredMatrices: Drop extreme-scale hardening from Vandermonde and Cauchy 963fffeb0 - NVHPC: Fix the nvc++ build 19090299a - Core: Use k-masked packet segments on AVX-512 and predicated segments on SVE 062290ffb - Tests: Guard pabs and pmadd/pmsub/pnmadd with packet traits in packetmath 210e0a609 - Core: Return the cached outer stride from direct-access Block 3cf994918 - CI: Build the AVX512-FP16 tests with clang-19 bbefaffa6 - Tests: Compile split test parts with only the functions they call 8c653283e - SME: Keep pack_direct in streaming mode under GCC c3123dc95 - BDCSVD: Cover the bidiagonal entry point in the switch-size test ca0565a17 - CI: Pass the SME -march through CMAKE_CXX_FLAGS e4505f831 - SME: Move the streaming primitives into arch/SME/PacketMath.h 1bd7cb0d1 - CI: Add an nvhpc-tests label for the NVHPC jobs 48b196e67 - Core: Drop constexpr from the noinline complex sqrt fallback 541f05781 - BDCSVD: Reallocate after setSwitchSize dae92cd3a - BLAS: Add ILP64 (64-bit integer) build and test support b79499c6f - AltiVec: Add pcmp_lt for Packet2l, fix the POWER7 build, and name the POWER8 check 6accef0ca - Core: Default-initialize storage in the expression constructors 0011719b1 - CMake: Disable if-conversion in the AVX512-FP16 tests on affected GCC 47bf0b657 - Tensor: Reject u == 0 in the normal random generator 859c035ba - Tests: Compile cheap split-test parts together d22c0df89 - Tests: Build redux_bounded_compile_vectorized with the job's flags d9ec2c9b9 - GPU: Copy views and adopt from a Context in the dense solvers' rvalue overloads ae47f25fb - CI: Isolate Windows sccache daemon port by CI_JOB_ID and ensure script teardown 3d836af2d - Tests: Stop special_functions from failing on cancellation ff16afbb1 - GPU: Give cuBLAS aligned copies of complex host scalars cbda2af3e - Core: Fix vector shape validation when automatic resizing is disabled 077d29983 - Core: Clamp the exponent before vscalef in the AVX-512 pldexp 9a97f13f6 - CI: Stop stale sccache servers on Windows runners cd1a734f2 - Core: Preserve bfloat16 packet sign under FTZ and DAZ 93933c899 - Core: Split pldexp's scale into three factors and round it once 356cdfd6e - Core: Preserve float and double sign under DAZ/FTZ a63497bea - CI: Run the GPU jobs and the cuBLAS/cuSOLVER tests for GPU-module changes 493c3a3a8 - GPU: Check CUDA and library calls in release builds 8e08dc1f5 - Jacobi: Select bounded ratio operands before division 851458df8 - Core: Compute subnormal integer powers exactly on ARMv7 NEON 38c71b0cf - GPU: Reduce into a caller's DeviceScalar, and compute norm() from the dot product a55314c6b - GPU: Stop solver result getters from waiting on unrelated GPU work ab89b1ab1 - GPU: Fix complex device-scalar axpy/scal and empty-operand GEMM on DeviceMatrix 367032aa0 - Core: Add adjoint() to diagonal, permutation and transposition matrices d17bdf5e9 - Tests: Index the FFT reference DFT's twiddles instead of calling exp a44124c5a - Core: Preserve reduction bounds through unary expressions 86e6ae8e2 - SME: one disjoint part of the result per SME unit, minimum task size per scalar type a1afbf4dd - Core: Vectorize isZero() and isApproxToConstant() checks 04bfe2287 - Core: Simplify packet traversal for strided coefficient assignments 1bd8e6f4d - SME: thread cap at the unit count, in-place LHS, ZA packers, kernels for narrow and tiny results 4df51993a - StructuredMatrices: Make DPR1 deflation exact for equal poles and option-independent ef5a9d5c4 - Core: Refactor approximate comparisons and fix false positives d142522b7 - Core: Fix the TRSM padded solve's link error in unoptimized C++14 builds d1b87b4cd - Core: Solve whole TRSM diagonal blocks with the shared packet kernel 651678320 - Core: Respect resizing restrictions and streamline small product evaluation ff1458835 - CI: Use '-' prefix for MSVC compiler flags so sccache caches Windows builds 6d293af5e - Core: Add lazy sums of dense, triangular and diagonal operands with diagonal and permutation matrices 563e4f40a - Compute integer powers by double-word repeated squaring, real and complex ed56f0c1d - Eigenvalues: Keep subnormal couplings under FTZ on MSVC and fix the test build 92b9ed6b0 - StructuredMatrices: Apply alpha to x in non-finite sparse Kronecker products a80a9cd50 - CI: Fix the GPU README docs failure and add a docs-build MR label dd16a1d7f - CI: Run the lint checks in one job on a prebuilt lint image 38295af54 - Core: Fix RealView coefficient references and IndexedView/RealView data() constness 0ec724769 - CI: Run clang-tidy on a prebuilt, pruned Ubuntu image af8d01b89 - Tests: Use explicit expected values for complex rsqrt corner cases ac940d63c - CI: Build the Windows CUDA tests through ccache, not sccache 9c7cebf31 - Core: Keep pldexp's scale factors from being reassociated under fast math ac63c7844 - CI: Pass failtest compiles through a missing launcher de177ddbb - CI: Provision sccache on Linux test runners for failtest jobs a7a0e35ee - Tests: Bound the scaled_permutation sum check by the terms, not the result 5fac38847 - Householder: Multiply a HouseholderSequence by a row vector 8746be0f2 - Core: Silence GCC 10 loop warnings in slice-vectorized reductions 91c8a4d8b - Householder: Apply a one-row reflector from the left 2dce125ec - CI: Fix sccache GCS credential JSON key and mask Windows token dump 53c33b64b - Core: Fix strides reported by vector IndexedView and strided RealView 9d194e7ad - Householder: Multiply HouseholderSequence by diagonal, triangular and permutation matrices 7681b6b7a - Core: Add permutation products with triangular and self-adjoint views 8a070f3e1 - CI: Add sccache with GCS remote backend with ccache fallback. 4b0b97830 - Core: Accelerate real vector operations with SME2 96aad0baa - Core: Fix scalar handling in mixed and self-adjoint products 59c28e04c - GPU: run Eigen's ConjugateGradient on the GPU types, with solveWithGuessInPlace 19292c62a - Docs: Fix the unresolved BSR section link in the GPU README a1a19e6b2 - Core, SVD, StructuredMatrices: Scale all-subnormal data exactly on flush-to-zero hardware 4cba415f3 - CI: Pre-bake curl, ca-certificates, and python3 into smoke build Dockerfiles 087757ad5 - Docs: Use Doxygen inline-math delimiters in ScaledPermutationMatrix.h e89236d8f - GPU: Add cuSPARSE BSR products for BlockSparseMatrix d30a9e6d3 - Core: Fix reshape traversal order and unsafe packet assignments 6f4f522f5 - EventCount: Fix signal loss in CancelWait b346103bd - Docs: Treat proprietary software as a black box eef236cb4 - SME: numext::round_down for the non-streaming panel peels 776f395d2 - Core: Add ScaledPermutationMatrix, the product of a permutation and a diagonal matrix ec8593a7d - Fix deprecated enum-enum conversion warning cba8148e3 - Core: Add products of two triangular or self-adjoint views 45e88a56d - Core: Harden the structured-product dispatch for shapes without a kernel 34e0e0793 - Eigenvalues: Reorthogonalize globally unresolved block shifts f26a444af - Core: Fix implicit Matrix conversions in std::variant d5a19d2ca - Fix GCC array-bounds warnings in Core, Sparse and Tensor ab4f7316d - CI: Auto-retry infrastructure drops on runner loss or preemption 14e4b5405 - Jacobi: Use power-of-two scaling and preserve subnormal real Givens rotations d32e9702b - SME: NEON path for small blocks, 64-byte packed panels, generic crossovers 647bc59a9 - Core: Fix scalar rewrites of structured products 8c6b5162b - Tests: Fix redux_12 and structured_kronecker_12 on 32-bit ARM 82a7cb22a - Agents: Present benchmark measurements as tables in merge request descriptions 47a78413d - Eigen: Standardize LAPACK helper routines into native C++ implementations 399d29312 - StructuredMatrices: Accept sparse factors in KroneckerOperator eddc25291 - SparseLU: Divide by subnormal pivots instead of forming their reciprocal 1b5286a2c - MSVC: Fix complex-component evaluator and tensor block-view builds b5c3e3da6 - Core: Use four accumulators in the vectorized reductions c7c04d737 - Core: Stabilize scalar complex square roots 7051b1f9a - Tests: Materialize the random vector before makeHouseholder in householder_essential_expressions 46a2d0c56 - Eigenvalues: Share and harden direct self-adjoint solver scaling 3ca5ab304 - CI: Allow runner-level CCACHE_* overrides via EIGEN_CI_CCACHE_* defaults e3c8cb150 - Core: Respect packet access in in-place transpose 1eef8230d - Core: Fix reverse row and column iterator indexing 04cc35554 - Core: Fix packet twoprod residuals and correct its error bound a076632bf - Performance: Avoid temporaries in non-aliasing product assignments c1a957f96 - Core: Share inner product dispatch and retain a tiny-vector path 59d9e069a - Tensor: Fix the GPU warnings that overflow the build log 0039ee376 - CI: Run 128-bit SVE tests natively on dedicated ARM runner 33fd96d09 - Docs: Re-attach the Transform::rotate Doxygen block and fix the GMRES \param name 93a193c58 - Core: Vectorize triangular solves across right-hand sides e1c311414 - Core: Enable linear counting and simplify visitor unrolling e7e5178d1 - Core: Exclude unreachable wide reductions for bounded expressions 2396b9dc2 - Core: Keep complex and class operands of the PowerPC GCC barrier in memory 02f77fd76 - CI: Skip the ccache round trip when an affected build compiles nothing ff285117c - Tests: Check backward error in dontalign QR solves 94f0b704f - Core: Reject Boolean subtraction and negation 8f8d4ed4c - Geometry: Support mixed-scalar operations in Transform::translate, pretranslate, and toRotationMatrix dd755f75e - Householder: Suppress false-positive -Warray-bounds warnings in BlockHouseholder bbcff8c67 - Eigenvalues: Avoid double promotion in scaling test 2800561ee - Core: Simplify and vectorize permutation operations 2a57319e6 - CMake: Compile GPU tests through CUDA and HIP language support 60c5525bb - Eigenvalues: Reuse QZ workspaces during compute 3f21c28a5 - CI: Parallelize the failtest suite and run native x86-64 tests on the large runner 154a40bd4 - Tensor: Avoid copying strided block operands 2002cae43 - Core: Avoid -Wfloat-equal warnings in SafeScaling for long double 2e7712e63 - CI: Use saas-linux-medium-amd64 for smoke build jobs da8cda1da - Core: Vectorize complex component reductions a14735e7e - Core: Add move support for DiagonalMatrix 4990b8359 - Eigenvalues: Break stalled ComplexQZ iterations with exceptional shifts 20b41e40b - QR: Document transposed solve limits and condition wide tests 99f1cccfd - Tests: Bound erfc comparisons in the subnormal range cd38c5ddc - Core/QR: Vectorize complex partial norms and simplify QR initialization 6a0625b84 - Sparse: Share packet scatter updates in Cholesky and LU 1373c7a36 - CI: Build only the affected selection in the SVE affected jobs 2298fbd89 - CI: Raise the SME full build timeout to 3h 20f5a26ba - Core: Remove misc directory and simplify LU expressions 15f227178 - Eigenvalues: Avoid cache-conflicting strides in RealSchur workspaces 74601a9e3 - CI: Log machine configuration before builds and tests 4a6bf607a - Eigenvalues: Use safe scaling in tridiagonal eigensolvers caac397f4 - Tensor: Enable double outer reductions with native sum atomics 27b0c620e - Tests: Allow for MSVC expm1 reference error 8facea8ba - SVD: Pack active secular terms for contiguous evaluation 6e86f64a0 - Core: Consolidate structured matrix norm utilities 5ad7d1788 - SVD: Reuse the saved row across blocked Jacobi rotations ec5876775 - SVD: Reuse operands while constructing singular vectors a4d943d1f - CI: Cross-compile the aarch64 builds on amd64 219e6c5a3 - HIP: Build tests for the requested GPU architectures 0f96e0f16 - Tests: Bound SVD normal-equation checks by backward error 7ec23fec7 - Core: Fix diagonal strides and subnormal identity scaling 68af15188 - Benchmarks: Add GPU kernel timing and validated baselines 50c7c77f3 - Tensor: Query attributes for the stream's owning GPU 4eaf42d05 - Docs: Ground performance changes in cost models and evidence ddf4a5479 - Eigenvalues: Use safe scaling in the iterative self-adjoint solver 3f4260bfe - Eigenvalues: Fix inconsistent nonfinite input status in in-place eigensolvers 5df2459f1 - Polynomials: Refine PolynomialSolver roots with Ehrlich-Aberth iterations 99baace37 - Eigenvalues: Use power-of-two scaling in Schur decompositions 6a026803b - CI: Cap parallel compiles in the aarch64 gcc-10 affected build e5b838361 - Geometry: Work around an LLVM ARM load/store miscompile in vectorwiseop 37e72f8df - Tests: Remove duplicate and near-duplicate test coverage 3bda1a485 - Warings: Fix some compiler warnings 133b7aa9d - StructuredMatrices: Vectorize scaling and reuse spectral work f0a977a96 - Tests: Work around an LLVM ARM load/store miscompile in vectorwiseop 0b9759a73 - Eigenvalues: Support inplace decomposition through Ref<> c0d6e9944 - GPU: Fix packet capabilities and simplify half operations 7a6afd466 - Geometry: Restore 4D vector assignment delegation in AngleAxis::operator= b17fadee5 - Core: Avoid -Wfloat-equal warnings in ceil_power_of_two for long double cae72577d - SparseCore: Silence unused variable warning under NDEBUG in triangular_solve_over_reach_iter 2d148dc46 - Tests: Add device packet-math coverage shared with the host suite 688bc7a5c - Core: Fix scalar complex GEMM accumulation on RVV PiperOrigin-RevId: 995455097 Change-Id: Idbbc059f9c9e5e999ec6c62d97ec2a5392c935d5
diff --git a/Eigen/Core b/Eigen/Core index 50d1f5f..7851324 100644 --- a/Eigen/Core +++ b/Eigen/Core
@@ -276,11 +276,12 @@ #include "src/Core/arch/NEON/MathFunctions.h" #include "src/Core/arch/NEON/Complex.h" #elif defined EIGEN_VECTORIZE_SME -// Utilize NEON for non-GEMM kernels. +// Keep NEON packet types outside the specialized streaming kernels. #include "src/Core/arch/NEON/PacketMath.h" #include "src/Core/arch/NEON/TypeCasting.h" #include "src/Core/arch/NEON/MathFunctions.h" #include "src/Core/arch/NEON/Complex.h" +#include "src/Core/arch/SME/PacketMath.h" #elif defined EIGEN_VECTORIZE_LSX #include "src/Core/arch/LSX/PacketMath.h" #include "src/Core/arch/LSX/TypeCasting.h" @@ -408,7 +409,7 @@ #include "src/Core/Redux.h" #include "src/Core/Visitor.h" #include "src/Core/FindCoeff.h" -#include "src/Core/Fuzzy.h" +#include "src/Core/ApproxComparisons.h" #include "src/Core/Swap.h" #include "src/Core/CommaInitializer.h" #include "src/Core/GeneralProduct.h" @@ -454,6 +455,7 @@ #include "src/Core/arch/SVE/GeneralBlockPanelKernel.h" #elif defined EIGEN_VECTORIZE_SME #include "src/Core/arch/SME/GeneralBlockPanelKernel.h" +#include "src/Core/arch/SME/VectorKernels.h" #elif defined EIGEN_VECTORIZE_LSX #include "src/Core/arch/LSX/GeneralBlockPanelKernel.h" #elif defined EIGEN_VECTORIZE_RVV10
diff --git a/Eigen/SparseCore b/Eigen/SparseCore index a3fd7a9..dcae114 100644 --- a/Eigen/SparseCore +++ b/Eigen/SparseCore
@@ -62,7 +62,7 @@ #include "src/SparseCore/TriangularReachSolver.h" #include "src/SparseCore/TriangularSolver.h" #include "src/SparseCore/SparsePermutation.h" -#include "src/SparseCore/SparseFuzzy.h" +#include "src/SparseCore/SparseApproxComparisons.h" #include "src/SparseCore/SparseSolverBase.h" #include "src/SparseCore/SparsityPatternRef.h"
diff --git a/Eigen/src/Cholesky/LLT.h b/Eigen/src/Cholesky/LLT.h index 64340d4..e4ffac8 100644 --- a/Eigen/src/Cholesky/LLT.h +++ b/Eigen/src/Cholesky/LLT.h
@@ -411,11 +411,13 @@ static Index blocked(MatrixType& m) { eigen_assert(m.rows() == m.cols()); Index size = m.rows(); - if (size < 32) return unblocked(m); + // The unblocked kernel walks row k of the factor, which strides through memory: it wins up to about 48 columns, + // and beyond that 16-column blocks keep those walks short while amortizing the level-3 updates. + if (size <= 48) return unblocked(m); Index blockSize = size / 8; blockSize = (blockSize / 16) * 16; - blockSize = (std::min)((std::max)(blockSize, Index(8)), Index(128)); + blockSize = (std::min)((std::max)(blockSize, Index(16)), Index(128)); for (Index k = 0; k < size; k += blockSize) { // partition the matrix:
diff --git a/Eigen/src/Core/ApproxComparisons.h b/Eigen/src/Core/ApproxComparisons.h new file mode 100644 index 0000000..785a630 --- /dev/null +++ b/Eigen/src/Core/ApproxComparisons.h
@@ -0,0 +1,464 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com> +// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// SPDX-License-Identifier: MPL-2.0 + +#ifndef EIGEN_APPROX_COMPARISONS_H +#define EIGEN_APPROX_COMPARISONS_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +// Keep custom scalars on the algebraic path: exponent scaling requires binary floating-point arithmetic. +template <typename Scalar> +struct use_scaled_comparison + : bool_constant<(std::is_floating_point<Scalar>::value && std::numeric_limits<Scalar>::radix == 2) || + std::is_same<Scalar, half>::value || std::is_same<Scalar, bfloat16>::value> {}; + +template <typename RealScalar> +struct use_scaled_comparison<std::complex<RealScalar>> : use_scaled_comparison<RealScalar> {}; + +// A nonnegative magnitude fraction * 2^exponent, including norms larger than the scalar range. Positive finite +// magnitudes order lexicographically by (exponent, fraction), the member order. +template <typename RealScalar> +struct comparison_magnitude { + int exponent = 0; + RealScalar fraction; + + EIGEN_DEVICE_FUNC explicit comparison_magnitude(const RealScalar& value) : fraction(value) { + if (value > RealScalar(0) && value <= NumTraits<RealScalar>::highest()) { + EIGEN_USING_STD(frexp); + fraction = frexp(value, &exponent); + } + } + + template <typename OtherRealScalar> + EIGEN_DEVICE_FUNC explicit comparison_magnitude(const comparison_magnitude<OtherRealScalar>& value) + : exponent(value.exponent), fraction(RealScalar(value.fraction)) {} + + EIGEN_DEVICE_FUNC void multiply(const RealScalar& value) { + const comparison_magnitude factor(value); + const comparison_magnitude product(fraction * factor.fraction); + fraction = product.fraction; + exponent += factor.exponent + product.exponent; + } + + EIGEN_DEVICE_FUNC bool isFinite() const { return fraction <= RealScalar(1); } +}; + +template <typename RealScalar> +EIGEN_DEVICE_FUNC bool operator<=(const comparison_magnitude<RealScalar>& x, + const comparison_magnitude<RealScalar>& y) { + if (!x.isFinite()) return false; + // Zero and non-finite magnitudes carry no exponent. + if (!y.isFinite() || x.fraction == RealScalar(0) || y.fraction == RealScalar(0)) return x.fraction <= y.fraction; + return x.exponent < y.exponent || (x.exponent == y.exponent && x.fraction <= y.fraction); +} + +template <typename RealScalar, typename OtherRealScalar> +EIGEN_DEVICE_FUNC bool operator<=(const comparison_magnitude<RealScalar>& x, + const comparison_magnitude<OtherRealScalar>& y) { + using Common = std::common_type_t<RealScalar, OtherRealScalar>; + return comparison_magnitude<Common>(x) <= comparison_magnitude<Common>(y); +} + +// Types whose data(), rowStride() and colStride() address every coefficient. DirectAccessBit alone does not promise +// this: a single-row IndexedView of a column-major matrix and the RealView of a strided complex block report strides +// that do not. +template <typename X> +struct scaled_comparison_viewable : std::false_type {}; + +template <typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols> +struct scaled_comparison_viewable<Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols>> : std::true_type {}; + +template <typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols> +struct scaled_comparison_viewable<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols>> : std::true_type {}; + +template <typename PlainObjectType, int Options, typename StrideType> +struct scaled_comparison_viewable<Map<PlainObjectType, Options, StrideType>> : std::true_type {}; + +template <typename PlainObjectType, int Options, typename StrideType> +struct scaled_comparison_viewable<Ref<PlainObjectType, Options, StrideType>> : std::true_type {}; + +template <typename XprType, int BlockRows, int BlockCols, bool InnerPanel> +struct scaled_comparison_viewable<Block<XprType, BlockRows, BlockCols, InnerPanel>> + : scaled_comparison_viewable<std::remove_const_t<XprType>> {}; + +template <typename XprType> +struct scaled_comparison_viewable<Transpose<XprType>> : scaled_comparison_viewable<std::remove_const_t<XprType>> {}; + +template <typename XprType> +struct scaled_comparison_viewable<ArrayWrapper<XprType>> : scaled_comparison_viewable<std::remove_const_t<XprType>> {}; + +template <typename XprType> +struct scaled_comparison_viewable<MatrixWrapper<XprType>> : scaled_comparison_viewable<std::remove_const_t<XprType>> {}; + +// The scaled path reduces each operand several times, so it runs on at most three view types per scalar type rather +// than being instantiated for every operand expression. A viewable operand with direct access is viewed in place, +// keeping packet access when its inner stride is one; any other operand, including a lazy expression or a shape that +// hides operator*(Scalar) as Homogeneous does, is evaluated first, which allocates only for dynamic sizes and for +// fixed sizes above EIGEN_STACK_ALLOCATION_LIMIT. +template <typename X, bool DirectAccess = has_direct_access<X>::value && scaled_comparison_viewable<X>::value, + bool UnitInnerStride = inner_stride_at_compile_time<X>::value == 1> +struct scaled_comparison_operand { + using View = + Map<const Matrix<typename X::Scalar, Dynamic, Dynamic, X::IsRowMajor ? RowMajor : ColMajor>, 0, OuterStride<>>; + EIGEN_DEVICE_FUNC explicit scaled_comparison_operand(const X& x) + : view(x.data(), x.rows(), x.cols(), OuterStride<>(x.outerStride())) {} + View view; +}; + +template <typename X> +struct scaled_comparison_operand<X, true, false> { + using View = Map<const Matrix<typename X::Scalar, Dynamic, Dynamic>, 0, Stride<Dynamic, Dynamic>>; + EIGEN_DEVICE_FUNC explicit scaled_comparison_operand(const X& x) + : view(x.data(), x.rows(), x.cols(), Stride<Dynamic, Dynamic>(x.colStride(), x.rowStride())) {} + View view; +}; + +template <typename X, bool UnitInnerStride> +struct scaled_comparison_operand<X, false, UnitInnerStride> { + // A fixed capacity the stack limit would reject makes the comparison fail to compile for an operand the caller + // never stores, so the temporary takes it only where DenseStorage accepts it. + static constexpr bool FixedCapacity = + EIGEN_STACK_ALLOCATION_LIMIT == 0 || + (X::MaxSizeAtCompileTime != Dynamic && + std::ptrdiff_t(X::MaxSizeAtCompileTime) * std::ptrdiff_t(sizeof(typename X::Scalar)) <= + std::ptrdiff_t(EIGEN_STACK_ALLOCATION_LIMIT)); + // Column-major unless it holds at most one row, where Matrix requires row-major storage. + using Plain = + Matrix<typename X::Scalar, Dynamic, Dynamic, + X::MaxRowsAtCompileTime == 1 && X::MaxColsAtCompileTime != 1 ? RowMajor : ColMajor, + FixedCapacity ? X::MaxRowsAtCompileTime : Dynamic, FixedCapacity ? X::MaxColsAtCompileTime : Dynamic>; + using View = typename scaled_comparison_operand<Plain>::View; + EIGEN_DEVICE_FUNC explicit scaled_comparison_operand(const X& x) + : value(x.matrix()), view(scaled_comparison_operand<Plain>(value).view) {} + Plain value; + View view; +}; + +template <typename Components> +EIGEN_DEVICE_FUNC typename Components::Scalar scaled_comparison_max_coeff(const Components& components) { + using RealScalar = typename Components::Scalar; + if (components.size() == 0) return RealScalar(0); + return safe_scaling<RealScalar>::recover_flushed_max_coeff(components, + components.cwiseAbs().template maxCoeff<PropagateNaN>()); +} + +template <typename Derived> +EIGEN_DEVICE_FUNC comparison_magnitude<typename stable_norm_accumulator<typename Derived::RealScalar>::type> +scaled_comparison_norm_impl(const MatrixBase<Derived>& matrix) { + using RealScalar = typename stable_norm_accumulator<typename Derived::RealScalar>::type; + const auto& realComponents = matrix.realView(); + const auto& components = realComponents.template cast<RealScalar>(); + const RealScalar scale = scaled_comparison_max_coeff(components); + // Classify first so NaNs do not reach the ordered comparison. + if (!(numext::isfinite)(scale) || !(scale > RealScalar(0))) return comparison_magnitude<RealScalar>(scale); + RealScalar squaredNorm = RealScalar(0); + const auto factors = safe_scaling<RealScalar>::with_scaled( + components, scale, [&](const auto& scaled) { squaredNorm = scaled.squaredNorm(); }); + comparison_magnitude<RealScalar> result(numext::sqrt(squaredNorm)); + result.multiply(factors.scale); + return result; +} + +template <typename X, typename Y> +EIGEN_DEVICE_FUNC comparison_magnitude<typename stable_norm_accumulator<typename X::RealScalar>::type> +scaled_comparison_distance_impl(const MatrixBase<X>& matrixX, const MatrixBase<Y>& matrixY) { + using Accumulator = typename stable_norm_accumulator<typename X::RealScalar>::type; + using WideScalar = + std::conditional_t<NumTraits<typename X::Scalar>::IsComplex || NumTraits<typename Y::Scalar>::IsComplex, + std::complex<Accumulator>, Accumulator>; + const auto& wideX = matrixX.template cast<WideScalar>(); + const auto& wideY = matrixY.template cast<WideScalar>(); + // FTZ flushes a subnormal difference of normal operands. Scaling a maximum M < 1 up by a power of two first loses + // only differences below M * min; scaling larger operands down could underflow their small components. + const Accumulator maxCoeff = numext::mini( + Accumulator(1), + numext::maxi(scaled_comparison_max_coeff(wideX.realView()), scaled_comparison_max_coeff(wideY.realView()))); + const safe_scaling_factors<Accumulator> factors = supports_power_of_two_scaling<Accumulator>::value + ? safe_scaling<Accumulator>::compute_floor_factors(maxCoeff) + : safe_scaling_factors<Accumulator>(); + // Both calls share one expression type, so scaled_comparison_norm_impl is instantiated once. + auto difference = scaled_comparison_norm_impl(wideX * factors.invScale - wideY * factors.invScale); + if (!difference.isFinite()) { + // Finite operands can overflow on subtraction; halving first keeps every component representable. + const Accumulator halfInvScale = factors.invScale * Accumulator(0.5); + difference = scaled_comparison_norm_impl(wideX * halfInvScale - wideY * halfInvScale); + difference.multiply(Accumulator(2)); + } + difference.multiply(factors.scale); + return difference; +} + +template <typename Derived> +EIGEN_DEVICE_FUNC comparison_magnitude<typename stable_norm_accumulator<typename Derived::RealScalar>::type> +scaled_comparison_norm(const Derived& x) { + return scaled_comparison_norm_impl(scaled_comparison_operand<Derived>(x).view); +} + +template <typename X, typename Y> +EIGEN_DEVICE_FUNC comparison_magnitude<typename stable_norm_accumulator<typename X::RealScalar>::type> +scaled_comparison_distance(const X& x, const Y& y) { + return scaled_comparison_distance_impl(scaled_comparison_operand<X>(x).view, scaled_comparison_operand<Y>(y).view); +} + +// Coefficients widened to the stable-norm accumulator, in which ordinary comparisons square and sum. The cast is the +// identity for float and double; half and bfloat16 widen exactly to float, where no half square underflows. +template <typename X> +using approx_comparison_wide_t = + std::conditional_t<NumTraits<typename X::Scalar>::IsComplex, + std::complex<typename stable_norm_accumulator<typename X::RealScalar>::type>, + typename stable_norm_accumulator<typename X::RealScalar>::type>; + +template <typename Scalar, bool = use_scaled_comparison<Scalar>::value> +struct approx_comparison_impl { + using RealScalar = typename NumTraits<Scalar>::Real; + + template <typename X, typename Y> + EIGEN_DEVICE_FUNC static bool isApprox(const X& x, const Y& y, const RealScalar& prec) { + return (x.matrix() - y.matrix()).cwiseAbs2().sum() <= + prec * prec * numext::mini(x.cwiseAbs2().sum(), y.cwiseAbs2().sum()); + } + + template <typename X, typename Y> + EIGEN_DEVICE_FUNC static bool isMuchSmallerThan(const X& x, const Y& y, const RealScalar& prec) { + return x.cwiseAbs2().sum() <= numext::abs2(prec) * y.cwiseAbs2().sum(); + } + + template <typename X> + EIGEN_DEVICE_FUNC static bool isMuchSmallerThan(const X& x, const RealScalar& y, const RealScalar& prec) { + return x.cwiseAbs2().sum() <= numext::abs2(prec * y); + } +}; + +template <typename Scalar> +struct approx_comparison_impl<Scalar, true> { + using RealScalar = typename NumTraits<Scalar>::Real; + using Accumulator = typename stable_norm_accumulator<RealScalar>::type; + template <typename Y> + using CommonAccumulator = + std::common_type_t<Accumulator, typename stable_norm_accumulator<typename Y::RealScalar>::type>; + + template <typename ValueScalar> + EIGEN_DEVICE_FUNC static typename stable_norm_accumulator<ValueScalar>::type squared_norm_lower_bound(Index size) { + using ValueAccumulator = typename stable_norm_accumulator<ValueScalar>::type; + // Squares accumulate in ValueAccumulator; below n * min / epsilon, flushed squares can affect the comparison. + return stable_normalization_normal_min<ValueAccumulator, ValueAccumulator>::run() / + NumTraits<ValueAccumulator>::epsilon() * ValueAccumulator(size); + } + + template <typename BoundScalar> + EIGEN_DEVICE_FUNC static bool safe_squared_norm(const BoundScalar& value, Index size) { + using Common = std::common_type_t<Accumulator, BoundScalar>; + return Common(value) >= Common(squared_norm_lower_bound<RealScalar>(size)) && + Common(value) <= Common(NumTraits<Accumulator>::highest()); + } + + template <typename X, typename Y> + EIGEN_DEVICE_FUNC static bool isApprox(const X& x, const Y& y, const RealScalar& prec) { + // Widening must not admit operands that cannot be subtracted, such as half and float. + EIGEN_CHECK_BINARY_COMPATIBILITY(scalar_difference_op<typename X::Scalar EIGEN_COMMA typename Y::Scalar>, + typename X::Scalar, typename Y::Scalar) + const auto& wideX = x.template cast<approx_comparison_wide_t<X>>(); + const auto& wideY = y.template cast<approx_comparison_wide_t<Y>>(); + const Accumulator x2 = wideX.cwiseAbs2().sum(); + const Accumulator y2 = wideY.cwiseAbs2().sum(); + const Accumulator minimum = numext::mini(x2, y2); + const Accumulator precision2 = Accumulator(prec) * Accumulator(prec); + const Accumulator bound = precision2 * minimum; + // Only the smaller norm enters the bound; overflow of the larger norm is harmless. + if (safe_squared_norm(bound, x.size()) && minimum >= squared_norm_lower_bound<RealScalar>(x.size()) && + precision2 >= squared_norm_lower_bound<RealScalar>(1)) + return (wideX.matrix() - wideY.matrix()).cwiseAbs2().sum() <= bound; + + return isApprox_scaled(x, y, prec); + } + + template <typename X, typename Y> + EIGEN_DEVICE_FUNC static bool isMuchSmallerThan(const X& x, const Y& y, const RealScalar& prec) { + using Common = CommonAccumulator<Y>; + typename nested_eval<X, 2>::type nested(x); + typename nested_eval<Y, 2>::type otherNested(y); + const Accumulator x2 = nested.template cast<approx_comparison_wide_t<X>>().cwiseAbs2().sum(); + const auto y2 = otherNested.template cast<approx_comparison_wide_t<Y>>().cwiseAbs2().sum(); + const Accumulator precision2 = numext::abs2(Accumulator(prec)); + const Common bound = Common(precision2) * Common(y2); + // A finite bound above the flushing error makes overflow/underflow of x2 harmless. + if (safe_squared_norm(bound, x.size()) && + Common(y2) >= Common(squared_norm_lower_bound<typename Y::RealScalar>(y.size())) && + precision2 >= squared_norm_lower_bound<RealScalar>(1)) + return Common(x2) <= bound; + return isMuchSmallerThan_scaled(nested, otherNested, prec); + } + + template <typename X> + EIGEN_DEVICE_FUNC static bool isMuchSmallerThan(const X& x, const RealScalar& y, const RealScalar& prec) { + typename nested_eval<X, 2>::type nested(x); + const Accumulator x2 = nested.template cast<approx_comparison_wide_t<X>>().cwiseAbs2().sum(); + const Accumulator bound = numext::abs2(Accumulator(prec) * Accumulator(y)); + if (safe_squared_norm(bound, x.size())) return x2 <= bound; + return isMuchSmallerThan_scaled(nested, y, prec); + } + + private: + // Keep exponent scaling from inhibiting inlining of ordinary comparisons. + template <typename X, typename Y> + EIGEN_DEVICE_FUNC static EIGEN_DONT_INLINE bool isApprox_scaled(const X& xExpr, const Y& yExpr, + const RealScalar& prec) { + const scaled_comparison_operand<X> x(xExpr); + const scaled_comparison_operand<Y> y(yExpr); + const auto nx = scaled_comparison_norm_impl(x.view); + const auto ny = scaled_comparison_norm_impl(y.view); + if (!nx.isFinite() || !ny.isFinite()) return false; + auto tolerance = nx <= ny ? nx : ny; + tolerance.multiply(numext::abs(Accumulator(prec))); + return scaled_comparison_distance_impl(x.view, y.view) <= tolerance; + } + + template <typename X, typename Y> + EIGEN_DEVICE_FUNC static EIGEN_DONT_INLINE bool isMuchSmallerThan_scaled(const X& x, const Y& y, + const RealScalar& prec) { + using Common = CommonAccumulator<Y>; + comparison_magnitude<Common> tolerance(scaled_comparison_norm(y)); + tolerance.multiply(numext::abs(Common(prec))); + return scaled_comparison_norm(x) <= tolerance; + } + + template <typename X> + EIGEN_DEVICE_FUNC static EIGEN_DONT_INLINE bool isMuchSmallerThan_scaled(const X& x, const RealScalar& y, + const RealScalar& prec) { + comparison_magnitude<Accumulator> tolerance(numext::abs(Accumulator(y))); + tolerance.multiply(numext::abs(Accumulator(prec))); + return scaled_comparison_norm(x) <= tolerance; + } +}; + +// Exponent scaling needs binary floating-point on both sides. +template <typename Derived, typename OtherDerived> +using approx_comparison_impl_t = + approx_comparison_impl<typename Derived::Scalar, use_scaled_comparison<typename Derived::Scalar>::value && + use_scaled_comparison<typename OtherDerived::Scalar>::value>; + +template <typename Derived, typename OtherDerived, bool is_integer = NumTraits<typename Derived::Scalar>::IsInteger> +struct isApprox_selector { + EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar& prec) { + typename internal::nested_eval<Derived, 2>::type nested(x); + typename internal::nested_eval<OtherDerived, 2>::type otherNested(y); + return approx_comparison_impl_t<Derived, OtherDerived>::isApprox(nested, otherNested, prec); + } +}; + +template <typename Derived, typename OtherDerived> +struct isApprox_selector<Derived, OtherDerived, true> { + EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar&) { + return x.matrix() == y.matrix(); + } +}; + +template <typename Derived, typename OtherDerived, bool is_integer = NumTraits<typename Derived::Scalar>::IsInteger> +struct isMuchSmallerThan_object_selector { + EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar& prec) { + return approx_comparison_impl_t<Derived, OtherDerived>::isMuchSmallerThan(x, y, prec); + } +}; + +template <typename Derived, typename OtherDerived> +struct isMuchSmallerThan_object_selector<Derived, OtherDerived, true> { + EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived&, const typename Derived::RealScalar&) { + return x.matrix() == Derived::Zero(x.rows(), x.cols()).matrix(); + } +}; + +template <typename Derived, bool is_integer = NumTraits<typename Derived::Scalar>::IsInteger> +struct isMuchSmallerThan_scalar_selector { + EIGEN_DEVICE_FUNC static bool run(const Derived& x, const typename Derived::RealScalar& y, + const typename Derived::RealScalar& prec) { + return approx_comparison_impl<typename Derived::Scalar>::isMuchSmallerThan(x, y, prec); + } +}; + +template <typename Derived> +struct isMuchSmallerThan_scalar_selector<Derived, true> + : isMuchSmallerThan_object_selector<Derived, typename Derived::RealScalar, true> {}; + +} // end namespace internal + +/** \returns \c true if \c *this is approximately equal to \a other, within the precision + * determined by \a prec. + * + * \note This is a relative norm comparison. Two vectors \f$ v \f$ and \f$ w \f$ + * are considered to be approximately equal within precision \f$ p \f$ if + * \f[ \Vert v - w \Vert \leqslant p\,\min(\Vert v\Vert, \Vert w\Vert). \f] + * For matrices, the comparison is done using the Hilbert-Schmidt norm (aka Frobenius norm + * L2 norm). + * + * \a prec is a relative tolerance chosen by the caller, not a bound on floating-point rounding error. + * The default, NumTraits<Scalar>::dummy_precision(), does not account for the operation, dimensions, + * or conditioning of the problem. Choose an explicit tolerance when checking numerical accuracy. + * A norm comparison also does not bound the relative error of each coefficient. + * + * \note Because of the multiplicativeness of this comparison, one can't use this function + * to check whether \c *this is approximately equal to the zero matrix or vector. + * Indeed, \c isApprox(zero) returns false unless \c *this itself is exactly the zero matrix + * or vector. If you want to test whether \c *this is small relative to a reference norm, + * use isMuchSmallerThan() instead. + * + * \sa isMuchSmallerThan(), isZero() + */ +template <typename Derived> +template <typename OtherDerived> +EIGEN_DEVICE_FUNC constexpr bool DenseBase<Derived>::isApprox(const DenseBase<OtherDerived>& other, + const RealScalar& prec) const { + return internal::isApprox_selector<Derived, OtherDerived>::run(derived(), other.derived(), prec); +} + +/** \returns \c true if the norm of \c *this is much smaller than \a other, + * within the precision determined by \a prec. + * + * \note This is a relative norm comparison. A vector \f$ v \f$ is + * considered to be much smaller than \f$ x \f$ within precision \f$ p \f$ if + * \f[ \Vert v \Vert \leqslant p\,\vert x\vert. \f] + * + * For matrices, the comparison is done using the Hilbert-Schmidt norm. For this reason, + * the value of the reference scalar \a other should come from the Hilbert-Schmidt norm + * of a reference matrix of same dimensions. + * + * \sa isApprox(), isMuchSmallerThan(const DenseBase<OtherDerived>&, RealScalar) const + */ +template <typename Derived> +EIGEN_DEVICE_FUNC constexpr bool DenseBase<Derived>::isMuchSmallerThan(const typename NumTraits<Scalar>::Real& other, + const RealScalar& prec) const { + return internal::isMuchSmallerThan_scalar_selector<Derived>::run(derived(), other, prec); +} + +/** \returns \c true if the norm of \c *this is much smaller than the norm of \a other, + * within the precision determined by \a prec. + * + * \note This is a relative norm comparison. A vector \f$ v \f$ is + * considered to be much smaller than a vector \f$ w \f$ within precision \f$ p \f$ if + * \f[ \Vert v \Vert \leqslant p\,\Vert w\Vert. \f] + * For matrices, the comparison is done using the Hilbert-Schmidt norm. + * + * \sa isApprox(), isMuchSmallerThan(const RealScalar&, RealScalar) const + */ +template <typename Derived> +template <typename OtherDerived> +EIGEN_DEVICE_FUNC constexpr bool DenseBase<Derived>::isMuchSmallerThan(const DenseBase<OtherDerived>& other, + const RealScalar& prec) const { + return internal::isMuchSmallerThan_object_selector<Derived, OtherDerived>::run(derived(), other.derived(), prec); +} + +} // end namespace Eigen + +#endif // EIGEN_APPROX_COMPARISONS_H
diff --git a/Eigen/src/Core/Array.h b/Eigen/src/Core/Array.h index 9ed22e3..43e6d5a 100644 --- a/Eigen/src/Core/Array.h +++ b/Eigen/src/Core/Array.h
@@ -110,10 +110,15 @@ * For dynamic-size matrices, creates an empty matrix of size 0. Does not allocate any array. Such a matrix * is called a null matrix. An existing matrix can also be turned into a null matrix by resizing it to 0. * + * Unless \c EIGEN_INITIALIZE_MATRICES_BY_ZERO or \c EIGEN_INITIALIZE_MATRICES_BY_NAN is defined, this constructor is + * defaulted, so value-initialization (<tt>Array a{};</tt>, <tt>Array()</tt>, or <tt>a()</tt> in a member + * initializer list) zero-initializes the whole object first, including the unused part of a fixed-maximum-size + * inline buffer. Default-initialization (<tt>Array a;</tt>) only default-initializes the coefficients. + * * \sa resize(Index,Index) */ #ifdef EIGEN_INITIALIZE_COEFFS - EIGEN_DEVICE_FUNC constexpr Array() : Base() { EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } + EIGEN_DEVICE_FUNC constexpr Array() { EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED } #else EIGEN_DEVICE_FUNC constexpr Array() = default; #endif
diff --git a/Eigen/src/Core/AssignEvaluator.h b/Eigen/src/Core/AssignEvaluator.h index 0008a94..f0ed534 100644 --- a/Eigen/src/Core/AssignEvaluator.h +++ b/Eigen/src/Core/AssignEvaluator.h
@@ -156,7 +156,14 @@ : Traversal == SliceVectorizedTraversal ? (MayUnrollInner ? InnerUnrolling : NoUnrolling) #endif : NoUnrolling; - static constexpr bool UsePacketSegment = has_packet_segment<PacketType>::value; + // Scalar tails for expressions bounded to <= 4 packets (runtime-sized blocks of small fixed matrices, as in the + // column steps of a 4x4 Cholesky): their results are reread almost at once, and a masked store does not forward to + // the following loads. That pays only while a tail is a few cheap coefficients: packets of <= 8 lanes, and a source + // cost below HugeCost (a lazy product with a runtime inner size computes a dot product per coefficient). + static constexpr bool UsePacketSegment = + has_packet_segment<PacketType>::value && + !(MaxSizeAtCompileTime != Dynamic && MaxSizeAtCompileTime <= 4 * int(unpacket_traits<PacketType>::size) && + unpacket_traits<PacketType>::size <= 8 && int(SrcEvaluator::CoeffReadCost) < HugeCost); #ifdef EIGEN_DEBUG_ASSIGN static void debug() { @@ -516,7 +523,13 @@ using tail_loop = unaligned_dense_assignment_loop<PacketType, Alignment, SrcAlignment, UsePacketSegment, false>; EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) { - const Index size = kernel.size(); + // When a small bound disables available packet segments, GCC's -Waggressive-loop-optimizations fires on the + // scalar head and tail loops unless they see that bound; the no-op clamp supplies it. Backends without segments + // keep the unclamped size and their existing codegen. + constexpr int MaxSize = Kernel::AssignmentTraits::MaxSizeAtCompileTime; + constexpr bool ClampToMaxSize = has_packet_segment<PacketType>::value && !UsePacketSegment && MaxSize != Dynamic; + eigen_assert(MaxSize == Dynamic || kernel.size() <= MaxSize); + const Index size = ClampToMaxSize ? numext::mini(kernel.size(), Index(MaxSize)) : kernel.size(); const Index alignedStart = DstIsAligned ? 0 : first_aligned<Alignment>(kernel.dstDataPtr(), size); head_loop::run(kernel, 0, alignedStart); @@ -654,8 +667,8 @@ } #if EIGEN_UNALIGNED_VECTORIZE - // The slice alignment offset varies from one outer index to the next. Unaligned stores with - // outer-invariant bounds beat chasing the aligned position of every slice. + // Unaligned stores with outer-invariant bounds avoid chasing each slice's alignment offset. + // Also used for statically aligned destinations to keep run() small enough to inline. using unaligned_tail_loop = unaligned_dense_assignment_loop<PacketType, Unaligned, Unaligned, UsePacketSegment, false>; @@ -672,9 +685,17 @@ #endif EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel) { - const Scalar* dst_ptr = kernel.dstDataPtr(); const Index innerSize = kernel.innerSize(); const Index outerSize = kernel.outerSize(); +#if EIGEN_UNALIGNED_VECTORIZE + EIGEN_IF_CONSTEXPR (DstIsAligned) { + // One loop instead of two keeps run() small enough for Clang to inline. Out of line, the + // evaluators' pointers and strides are reloaded after every packet store, which may alias them. + runUnaligned(kernel, innerSize, outerSize); + return; + } +#endif + const Scalar* dst_ptr = kernel.dstDataPtr(); const Index alignedStep = Alignable ? (PacketSize - kernel.outerStride() % PacketSize) % PacketSize : 0; Index alignedStart = ((!Alignable) || DstIsAligned) ? 0 : internal::first_aligned<Alignment>(dst_ptr, innerSize); @@ -896,10 +917,8 @@ Index dstCols = src.cols(); if (((dst.rows() != dstRows) || (dst.cols() != dstCols))) { #ifdef EIGEN_NO_AUTOMATIC_RESIZING - eigen_assert( - (dst.size() == 0 || (DstXprType::IsVectorAtCompileTime ? (dst.size() == src.size()) - : (dst.rows() == dstRows && dst.cols() == dstCols))) && - "Size mismatch. Automatic resizing is disabled because EIGEN_NO_AUTOMATIC_RESIZING is defined"); + eigen_assert(dst.size() == 0 && + "Size mismatch. Automatic resizing is disabled because EIGEN_NO_AUTOMATIC_RESIZING is defined"); if (dst.size() == 0) { dst.resize(dstRows, dstCols); }
diff --git a/Eigen/src/Core/Block.h b/Eigen/src/Core/Block.h index a160cbe..c995430 100644 --- a/Eigen/src/Core/Block.h +++ b/Eigen/src/Core/Block.h
@@ -385,9 +385,7 @@ } /** \sa MapBase::outerStride() */ - EIGEN_DEVICE_FUNC constexpr Index outerStride() const noexcept { - return internal::traits<BlockType>::HasSameStorageOrderAsXprType ? m_xpr.outerStride() : m_xpr.innerStride(); - } + EIGEN_DEVICE_FUNC constexpr Index outerStride() const noexcept { return m_outerStride; } EIGEN_DEVICE_FUNC constexpr StorageIndex startRow() const noexcept { return m_startRow.value(); }
diff --git a/Eigen/src/Core/CwiseNullaryOp.h b/Eigen/src/Core/CwiseNullaryOp.h index 218fac9..97aa10d 100644 --- a/Eigen/src/Core/CwiseNullaryOp.h +++ b/Eigen/src/Core/CwiseNullaryOp.h
@@ -307,16 +307,6 @@ return DenseBase<Derived>::NullaryExpr(Derived::SizeAtCompileTime, internal::equalspaced_op<Scalar>(low, step)); } -/** \returns true if all coefficients in this matrix are approximately equal to \a val, to within precision \a prec */ -template <typename Derived> -EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isApproxToConstant(const Scalar& val, const RealScalar& prec) const { - typename internal::nested_eval<Derived, 1>::type self(derived()); - for (Index j = 0; j < cols(); ++j) - for (Index i = 0; i < rows(); ++i) - if (!internal::isApprox(self.coeff(i, j), val, prec)) return false; - return true; -} - /** This is just an alias for isApproxToConstant(). * * \returns true if all coefficients in this matrix are approximately equal to \a value, to within precision \a prec */ @@ -520,23 +510,6 @@ return ZeroReturnType(RowsAtCompileTime, ColsAtCompileTime); } -/** \returns true if *this is approximately equal to the zero matrix, - * within the precision given by \a prec. - * - * Example: \include MatrixBase_isZero.cpp - * Output: \verbinclude MatrixBase_isZero.out - * - * \sa class CwiseNullaryOp, Zero() - */ -template <typename Derived> -EIGEN_DEVICE_FUNC bool DenseBase<Derived>::isZero(const RealScalar& prec) const { - typename internal::nested_eval<Derived, 1>::type self(derived()); - for (Index j = 0; j < cols(); ++j) - for (Index i = 0; i < rows(); ++i) - if (!internal::isMuchSmallerThan(self.coeff(i, j), static_cast<Scalar>(1), prec)) return false; - return true; -} - /** Sets all coefficients in this expression to zero. * * Example: \include MatrixBase_setZero.cpp
diff --git a/Eigen/src/Core/DenseBase.h b/Eigen/src/Core/DenseBase.h index 97703fb..9cdb42b 100644 --- a/Eigen/src/Core/DenseBase.h +++ b/Eigen/src/Core/DenseBase.h
@@ -513,13 +513,33 @@ template <int RowFactor, int ColFactor> EIGEN_DEVICE_FUNC const Replicate<Derived, RowFactor, ColFactor> replicate() const; + + /** + * \return an expression of the replication of \c *this + * + * The replication factors can be passed at run time as integers, or at compile time as Eigen::fix<N> or + * Eigen::fix<N>(n). In the latter case, \c n is the runtime value used when \c N equals Eigen::Dynamic, and must + * equal \c N otherwise. For example, with a fixed row factor \c NRows and a runtime column factor \c cols: + * \code mat.replicate(fix<NRows>, cols) \endcode + * + * \sa DenseBase::replicate(Index,Index) const, DenseBase::replicate<int,int>(), class Replicate, fix, fix<N>(int) + */ + template <typename NRowsType, typename NColsType> + EIGEN_DEVICE_FUNC const + Replicate<Derived, internal::get_fixed_value<NRowsType>::value, internal::get_fixed_value<NColsType>::value> + replicate(NRowsType nRows, NColsType nCols) const { + return Replicate<Derived, internal::get_fixed_value<NRowsType>::value, internal::get_fixed_value<NColsType>::value>( + derived(), internal::get_runtime_value(nRows), internal::get_runtime_value(nCols)); + } + /** * \return an expression of the replication of \c *this * * Example: \include MatrixBase_replicate_int_int.cpp * Output: \verbinclude MatrixBase_replicate_int_int.out * - * \sa VectorwiseOp::replicate(), DenseBase::replicate<int,int>(), class Replicate + * \sa VectorwiseOp::replicate(), DenseBase::replicate<int,int>(), DenseBase::replicate(NRowsType,NColsType) const, + * class Replicate */ // Code moved here due to a CUDA compiler bug EIGEN_DEVICE_FUNC const Replicate<Derived, Dynamic, Dynamic> replicate(Index rowFactor, Index colFactor) const {
diff --git a/Eigen/src/Core/DiagonalMatrix.h b/Eigen/src/Core/DiagonalMatrix.h index 989cbba..4979e14 100644 --- a/Eigen/src/Core/DiagonalMatrix.h +++ b/Eigen/src/Core/DiagonalMatrix.h
@@ -44,6 +44,8 @@ ColsAtCompileTime = DiagonalVectorType::SizeAtCompileTime, MaxRowsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime, MaxColsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime, + SizeAtCompileTime = internal::size_at_compile_time(RowsAtCompileTime, ColsAtCompileTime), + MaxSizeAtCompileTime = internal::size_at_compile_time(MaxRowsAtCompileTime, MaxColsAtCompileTime), IsVectorAtCompileTime = 0, Flags = NoPreferredStorageOrderBit }; @@ -107,6 +109,31 @@ return diagonal().cwiseInverse().asDiagonal(); } + using TransposeReturnType = DiagonalWrapper<const DiagonalVectorType>; + + /** \returns the transpose of \c *this, a diagonal matrix with the same diagonal. + * + * \sa conjugate(), adjoint() */ + EIGEN_DEVICE_FUNC const TransposeReturnType transpose() const { return diagonal().asDiagonal(); } + + using ConjugateReturnType = std::conditional_t< + NumTraits<Scalar>::IsComplex, + DiagonalWrapper<const CwiseUnaryOp<internal::scalar_conjugate_op<Scalar>, const DiagonalVectorType>>, + TransposeReturnType>; + + /** \returns the complex conjugate of \c *this; for real scalars, a diagonal matrix with the same diagonal. + * + * \sa transpose(), adjoint() */ + EIGEN_DEVICE_FUNC const ConjugateReturnType conjugate() const { return diagonal().conjugate().asDiagonal(); } + + using AdjointReturnType = ConjugateReturnType; + + /** \returns the adjoint (conjugate transpose) of \c *this. A diagonal matrix equals its transpose, so this is + * conjugate(). + * + * \sa transpose(), conjugate() */ + EIGEN_DEVICE_FUNC const AdjointReturnType adjoint() const { return conjugate(); } + using DiagonalScaleReturnType = DiagonalWrapper<const EIGEN_EXPR_BINARYOP_SCALAR_RETURN_TYPE( DiagonalVectorType, Scalar, internal::scalar_product_op)>; @@ -145,6 +172,39 @@ const DiagonalBase<OtherDerived>& other) const { return (diagonal() - other.diagonal()).asDiagonal(); } + + // Sums with a dense matrix are lazy: the diagonal is read through its index-based evaluator and no dense + // copy of it is formed, so `A + D` composes with the enclosing expression like `A + B` does. + + /** \returns the lazy sum of the dense matrix \a lhs and the diagonal matrix \a rhs */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC friend const EIGEN_CWISE_BINARY_RETURN_TYPE(OtherDerived, Derived, internal::scalar_sum_op) + operator+(const MatrixBase<OtherDerived>& lhs, const DiagonalBase & rhs) { + return EIGEN_CWISE_BINARY_RETURN_TYPE(OtherDerived, Derived, internal::scalar_sum_op)(lhs.derived(), rhs.derived()); + } + + /** \returns the lazy sum of the diagonal matrix \a lhs and the dense matrix \a rhs */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC friend const EIGEN_CWISE_BINARY_RETURN_TYPE(Derived, OtherDerived, internal::scalar_sum_op) + operator+(const DiagonalBase & lhs, const MatrixBase<OtherDerived>& rhs) { + return EIGEN_CWISE_BINARY_RETURN_TYPE(Derived, OtherDerived, internal::scalar_sum_op)(lhs.derived(), rhs.derived()); + } + + /** \returns the lazy difference of the dense matrix \a lhs and the diagonal matrix \a rhs */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC friend const EIGEN_CWISE_BINARY_RETURN_TYPE(OtherDerived, Derived, internal::scalar_difference_op) + operator-(const MatrixBase<OtherDerived>& lhs, const DiagonalBase & rhs) { + return EIGEN_CWISE_BINARY_RETURN_TYPE(OtherDerived, Derived, internal::scalar_difference_op)(lhs.derived(), + rhs.derived()); + } + + /** \returns the lazy difference of the diagonal matrix \a lhs and the dense matrix \a rhs */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC friend const EIGEN_CWISE_BINARY_RETURN_TYPE(Derived, OtherDerived, internal::scalar_difference_op) + operator-(const DiagonalBase & lhs, const MatrixBase<OtherDerived>& rhs) { + return EIGEN_CWISE_BINARY_RETURN_TYPE(Derived, OtherDerived, internal::scalar_difference_op)(lhs.derived(), + rhs.derived()); + } }; /** \class DiagonalMatrix @@ -449,6 +509,52 @@ using Shape = DiagonalShape; }; +/** \internal + * Index-based evaluator of a diagonal matrix, for coefficient-wise expressions that mix it with a dense + * operand. Off-diagonal coefficients are synthesized, so there is no linear, packet or direct access. + * Products with a diagonal matrix have dedicated evaluators and do not use this one. + */ +template <typename XprType> +struct diagonal_matrix_evaluator : evaluator_base<XprType> { + using DiagonalVectorType = typename XprType::DiagonalVectorType; + using Scalar = typename XprType::Scalar; + using CoeffReturnType = Scalar; + + static constexpr int CoeffReadCost = + int(evaluator<DiagonalVectorType>::CoeffReadCost) + int(NumTraits<Scalar>::AddCost); + static constexpr unsigned int Flags = 0; + static constexpr int Alignment = 0; + + EIGEN_DEVICE_FUNC explicit diagonal_matrix_evaluator(const XprType& xpr) : m_diagonal(xpr.diagonal()) { + EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost); + } + + EIGEN_DEVICE_FUNC Scalar coeff(Index row, Index col) const { return row == col ? m_diagonal.coeff(row) : Scalar(0); } + + // Linear access is requested only for vector-shaped operands (inner products), i.e. a 1x1 diagonal. + EIGEN_DEVICE_FUNC Scalar coeff(Index index) const { + eigen_assert(index == 0); + return m_diagonal.coeff(index); + } + + protected: + evaluator<DiagonalVectorType> m_diagonal; +}; + +template <typename Scalar_, int SizeAtCompileTime, int MaxSizeAtCompileTime> +struct evaluator<DiagonalMatrix<Scalar_, SizeAtCompileTime, MaxSizeAtCompileTime>> + : diagonal_matrix_evaluator<DiagonalMatrix<Scalar_, SizeAtCompileTime, MaxSizeAtCompileTime>> { + using XprType = DiagonalMatrix<Scalar_, SizeAtCompileTime, MaxSizeAtCompileTime>; + EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : diagonal_matrix_evaluator<XprType>(xpr) {} +}; + +template <typename DiagonalVectorType_> +struct evaluator<DiagonalWrapper<DiagonalVectorType_>> + : diagonal_matrix_evaluator<DiagonalWrapper<DiagonalVectorType_>> { + using XprType = DiagonalWrapper<DiagonalVectorType_>; + EIGEN_DEVICE_FUNC explicit evaluator(const XprType& xpr) : diagonal_matrix_evaluator<XprType>(xpr) {} +}; + struct Diagonal2Dense {}; template <> @@ -483,6 +589,199 @@ } }; +/*************************************************************************** + * Dense ?= lhs +/- rhs, where one operand is dense and the other has one structural nonzero in each column of the + * destination: the lazy CwiseBinaryOp keeps its type, but a direct assignment writes each column as two vectorized + * segments of the dense operand around that nonzero, instead of evaluating the sum coefficient by coefficient + * without packets. + ***************************************************************************/ + +// The three functors such an assignment can carry, and the sign with which they add the source. +template <typename Functor> +struct additive_assign_sign : std::integral_constant<int, 0> {}; +template <typename Scalar> +struct additive_assign_sign<assign_op<Scalar, Scalar>> : std::integral_constant<int, 1> {}; +template <typename Scalar> +struct additive_assign_sign<add_assign_op<Scalar, Scalar>> : std::integral_constant<int, 1> {}; +template <typename Scalar> +struct additive_assign_sign<sub_assign_op<Scalar, Scalar>> : std::integral_constant<int, -1> {}; + +template <typename Functor> +struct is_plain_assign : std::false_type {}; +template <typename Scalar> +struct is_plain_assign<assign_op<Scalar, Scalar>> : std::true_type {}; + +template <typename BinaryOp> +struct is_additive_binary_op : std::false_type {}; +template <typename Scalar> +struct is_additive_binary_op<scalar_sum_op<Scalar, Scalar>> : std::true_type {}; +template <typename Scalar> +struct is_additive_binary_op<scalar_difference_op<Scalar, Scalar>> : std::true_type {}; + +template <typename T> +struct is_dense_shape : std::is_same<typename evaluator_traits<T>::Shape, DenseShape> {}; +template <typename T> +struct is_diagonal_shape : std::is_same<typename evaluator_traits<T>::Shape, DiagonalShape> {}; +template <typename T> +struct is_default_product : std::false_type {}; +template <typename Lhs, typename Rhs> +struct is_default_product<Product<Lhs, Rhs, DefaultProduct>> : std::true_type {}; +// Specialized in PermutationMatrix.h: a permutation's dense expression counts as the structured operand of a sum +// with a diagonal matrix, not as its dense one. +template <typename T> +struct is_permutation_dense_xpr : std::false_type {}; + +template <typename BinaryOp> +struct is_difference_op : std::false_type {}; +template <typename Scalar> +struct is_difference_op<scalar_difference_op<Scalar, Scalar>> : std::true_type {}; + +// The block pass below also forms dst ?= +/-dense at the structured coefficients, then overwrites them. For signed +// integers that intermediate can overflow where the coefficient-wise dst ?= op(lhs, rhs) does not (-INT_MIN, or +// INT_MIN - 1 in INT_MIN - (1 - 1)), so they take the fast path only when the block pass is a copy. +template <typename Scalar, typename Functor, bool NegateDense> +struct dense_block_pass_cannot_overflow + : bool_constant<!(NumTraits<Scalar>::IsInteger && std::numeric_limits<Scalar>::is_signed) || + (is_plain_assign<Functor>::value && !NegateDense)> {}; + +// Mixed scalar types and functors other than =, += and -= stay on the generic coefficient-wise path. +template <typename Dst, typename DenseXpr, typename DiagonalXpr, typename Functor, bool NegateDense> +struct dense_diagonal_sum_fast_path + : bool_constant<is_dense_shape<DenseXpr>::value && !is_permutation_dense_xpr<DenseXpr>::value && + is_diagonal_shape<DiagonalXpr>::value && additive_assign_sign<Functor>::value != 0 && + std::is_same<typename Dst::Scalar, typename DenseXpr::Scalar>::value && + std::is_same<typename DenseXpr::Scalar, typename DiagonalXpr::Scalar>::value && + dense_block_pass_cannot_overflow<typename Dst::Scalar, Functor, NegateDense>::value> {}; + +// block ?= +/-dense block as one of =, = -, += and -=, chosen at compile time: bool has no negation or subtraction, +// so no other operator may be instantiated. +template <bool Assign, int Sign> +struct dense_block_update; +template <> +struct dense_block_update<true, 1> { + template <typename Block, typename DenseBlock> + EIGEN_DEVICE_FUNC static void run(Block&& block, const DenseBlock& dense) { + block = dense; + } +}; +template <> +struct dense_block_update<true, -1> { + template <typename Block, typename DenseBlock> + EIGEN_DEVICE_FUNC static void run(Block&& block, const DenseBlock& dense) { + block = -dense; + } +}; +template <> +struct dense_block_update<false, 1> { + template <typename Block, typename DenseBlock> + EIGEN_DEVICE_FUNC static void run(Block&& block, const DenseBlock& dense) { + block += dense; + } +}; +template <> +struct dense_block_update<false, -1> { + template <typename Block, typename DenseBlock> + EIGEN_DEVICE_FUNC static void run(Block&& block, const DenseBlock& dense) { + block -= dense; + } +}; + +// A diagonal as the structured operand: its nonzero in column k is at row k. +template <typename DiagonalXpr> +struct diagonal_column_nonzeros { + using Scalar = typename DiagonalXpr::Scalar; + EIGEN_DEVICE_FUNC explicit diagonal_column_nonzeros(const DiagonalXpr& diagonal) : m_diagonal(diagonal) {} + EIGEN_DEVICE_FUNC Index row(Index k) const { return k; } + EIGEN_DEVICE_FUNC Scalar value(Index k) const { return m_diagonal.coeff(k, k); } + evaluator<DiagonalXpr> m_diagonal; +}; + +// Column-major views of a row-major destination and its dense operand; a structured operand is only paired with a +// destination along whose columns it has one nonzero each, so after transposition it needs no view of its own. +template <bool Transpose> +struct column_major_view { + template <typename Xpr> + EIGEN_DEVICE_FUNC static Xpr& run(Xpr& xpr) { + return xpr; + } +}; +template <> +struct column_major_view<true> { + template <typename Xpr> + EIGEN_DEVICE_FUNC static auto run(Xpr& xpr) { + return xpr.transpose(); + } +}; + +// dst ?= op(lhs, rhs) in blocks of columns. For each column k of a block, with the structured operand's nonzero at +// row r, v(k) = func(dst(r, k), op(lhs(r, k), rhs(r, k))) is formed from coefficients read before the block is +// written; then block ?= +/-dense block, and dst(r, k) = v(k). So each structured coefficient gets the +// coefficient-wise value and grouping, also when the diagonal or the dense operand is read from dst; structural +// zeros are not added. +template <bool DenseOnLeft> +struct dense_structured_sum_assignment { + template <typename Dst, typename DenseXpr, typename Structured, typename BinaryOp, typename Functor> + EIGEN_DEVICE_FUNC static void run(Dst& dst, const DenseXpr& dense, const Structured& structured, const BinaryOp& op, + const Functor& func) { + using Scalar = typename Dst::Scalar; + constexpr bool kNegateDense = !DenseOnLeft && std::is_same<BinaryOp, scalar_difference_op<Scalar, Scalar>>::value; + constexpr int kSign = additive_assign_sign<Functor>::value * (kNegateDense ? -1 : 1); + constexpr Index kBlockColumns = 32; + using Update = dense_block_update<is_plain_assign<Functor>::value, kSign>; + const evaluator<DenseXpr> denseEval(dense); + Scalar values[kBlockColumns]; + for (Index j = 0; j < dst.cols(); j += kBlockColumns) { + const Index columns = numext::mini(Index(kBlockColumns), dst.cols() - j); + for (Index k = 0; k < columns; ++k) { + const Index r = structured.row(j + k); + const Scalar s = structured.value(j + k); + const Scalar d = denseEval.coeff(r, j + k); + EIGEN_IF_CONSTEXPR (!is_plain_assign<Functor>::value) { + values[k] = dst.coeff(r, j + k); + } + func.assignCoeff(values[k], DenseOnLeft ? op(d, s) : op(s, d)); + } + Update::run(dst.middleCols(j, columns), dense.middleCols(j, columns)); + for (Index k = 0; k < columns; ++k) { + dst.coeffRef(structured.row(j + k), j + k) = values[k]; + } + } + } + + template <typename Dst, typename SrcXprType, typename DenseXpr, typename Structured, typename Functor> + EIGEN_DEVICE_FUNC static void assign(Dst& dst, const SrcXprType& src, const DenseXpr& denseXpr, + const Structured& structured, const Functor& func) { + // Evaluates a product operand once, before dst is resized or written. + const typename nested_eval<DenseXpr, 1>::type dense(denseXpr); + resize_if_allowed(dst, src, func); + auto&& dstView = column_major_view<bool(Dst::IsRowMajor)>::run(dst); + run(dstView, column_major_view<bool(Dst::IsRowMajor)>::run(dense), structured, src.functor(), func); + } +}; + +template <typename DstXprType, typename BinaryOp, typename Lhs, typename Rhs, typename Functor> +struct Assignment<DstXprType, CwiseBinaryOp<BinaryOp, const Lhs, const Rhs>, Functor, Dense2Dense, + std::enable_if_t<is_additive_binary_op<BinaryOp>::value && + dense_diagonal_sum_fast_path<DstXprType, Lhs, Rhs, Functor, false>::value>> { + using SrcXprType = CwiseBinaryOp<BinaryOp, const Lhs, const Rhs>; + EIGEN_DEVICE_FUNC static void run(DstXprType& dst, const SrcXprType& src, const Functor& func) { + dense_structured_sum_assignment<true>::assign(dst, src, src.lhs(), diagonal_column_nonzeros<Rhs>(src.rhs()), func); + } +}; + +// A dense product on the right is left to the "xpr + product" rule above. +template <typename DstXprType, typename BinaryOp, typename Lhs, typename Rhs, typename Functor> +struct Assignment<DstXprType, CwiseBinaryOp<BinaryOp, const Lhs, const Rhs>, Functor, Dense2Dense, + std::enable_if_t<is_additive_binary_op<BinaryOp>::value && + dense_diagonal_sum_fast_path<DstXprType, Rhs, Lhs, Functor, + is_difference_op<BinaryOp>::value>::value && + !is_default_product<Rhs>::value>> { + using SrcXprType = CwiseBinaryOp<BinaryOp, const Lhs, const Rhs>; + EIGEN_DEVICE_FUNC static void run(DstXprType& dst, const SrcXprType& src, const Functor& func) { + dense_structured_sum_assignment<false>::assign(dst, src, src.rhs(), diagonal_column_nonzeros<Lhs>(src.lhs()), func); + } +}; + } // namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/FindCoeff.h b/Eigen/src/Core/FindCoeff.h index 5802ed9..d7f87b6 100644 --- a/Eigen/src/Core/FindCoeff.h +++ b/Eigen/src/Core/FindCoeff.h
@@ -108,9 +108,11 @@ } }; +// The packet loop needs pcmp_lt (and pcmp_lt_or_nan/pisnan for the NaN-aware functors); predux_max/predux_min +// fall back to a per-lane reduction, so HasMax/HasMin are not required. template <typename Scalar> struct min_max_traits { - static constexpr bool PacketAccess = packet_traits<Scalar>::Vectorizable; + static constexpr bool PacketAccess = packet_traits<Scalar>::Vectorizable && packet_traits<Scalar>::HasCmp; }; template <typename Scalar, int NaNPropagation> struct functor_traits<max_coeff_functor<Scalar, NaNPropagation>> : min_max_traits<Scalar> {};
diff --git a/Eigen/src/Core/Fuzzy.h b/Eigen/src/Core/Fuzzy.h deleted file mode 100644 index a802b9c..0000000 --- a/Eigen/src/Core/Fuzzy.h +++ /dev/null
@@ -1,133 +0,0 @@ -// This file is part of Eigen, a lightweight C++ template library -// for linear algebra. -// -// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com> -// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr> -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// SPDX-License-Identifier: MPL-2.0 - -#ifndef EIGEN_FUZZY_H -#define EIGEN_FUZZY_H - -// IWYU pragma: private -#include "./InternalHeaderCheck.h" - -namespace Eigen { - -namespace internal { - -template <typename Derived, typename OtherDerived, bool is_integer = NumTraits<typename Derived::Scalar>::IsInteger> -struct isApprox_selector { - EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar& prec) { - typename internal::nested_eval<Derived, 2>::type nested(x); - typename internal::nested_eval<OtherDerived, 2>::type otherNested(y); - return (nested.matrix() - otherNested.matrix()).cwiseAbs2().sum() <= - prec * prec * numext::mini(nested.cwiseAbs2().sum(), otherNested.cwiseAbs2().sum()); - } -}; - -template <typename Derived, typename OtherDerived> -struct isApprox_selector<Derived, OtherDerived, true> { - EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar&) { - return x.matrix() == y.matrix(); - } -}; - -template <typename Derived, typename OtherDerived, bool is_integer = NumTraits<typename Derived::Scalar>::IsInteger> -struct isMuchSmallerThan_object_selector { - EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar& prec) { - return x.cwiseAbs2().sum() <= numext::abs2(prec) * y.cwiseAbs2().sum(); - } -}; - -template <typename Derived, typename OtherDerived> -struct isMuchSmallerThan_object_selector<Derived, OtherDerived, true> { - EIGEN_DEVICE_FUNC static bool run(const Derived& x, const OtherDerived&, const typename Derived::RealScalar&) { - return x.matrix() == Derived::Zero(x.rows(), x.cols()).matrix(); - } -}; - -template <typename Derived, bool is_integer = NumTraits<typename Derived::Scalar>::IsInteger> -struct isMuchSmallerThan_scalar_selector { - EIGEN_DEVICE_FUNC static bool run(const Derived& x, const typename Derived::RealScalar& y, - const typename Derived::RealScalar& prec) { - return x.cwiseAbs2().sum() <= numext::abs2(prec * y); - } -}; - -template <typename Derived> -struct isMuchSmallerThan_scalar_selector<Derived, true> { - EIGEN_DEVICE_FUNC static bool run(const Derived& x, const typename Derived::RealScalar&, - const typename Derived::RealScalar&) { - return x.matrix() == Derived::Zero(x.rows(), x.cols()).matrix(); - } -}; - -} // end namespace internal - -/** \returns \c true if \c *this is approximately equal to \a other, within the precision - * determined by \a prec. - * - * \note The fuzzy compares are done multiplicatively. Two vectors \f$ v \f$ and \f$ w \f$ - * are considered to be approximately equal within precision \f$ p \f$ if - * \f[ \Vert v - w \Vert \leqslant p\,\min(\Vert v\Vert, \Vert w\Vert). \f] - * For matrices, the comparison is done using the Hilbert-Schmidt norm (aka Frobenius norm - * L2 norm). - * - * \note Because of the multiplicativeness of this comparison, one can't use this function - * to check whether \c *this is approximately equal to the zero matrix or vector. - * Indeed, \c isApprox(zero) returns false unless \c *this itself is exactly the zero matrix - * or vector. If you want to test whether \c *this is zero, use internal::isMuchSmallerThan(const - * RealScalar&, RealScalar) instead. - * - * \sa internal::isMuchSmallerThan(const RealScalar&, RealScalar) const - */ -template <typename Derived> -template <typename OtherDerived> -EIGEN_DEVICE_FUNC constexpr bool DenseBase<Derived>::isApprox(const DenseBase<OtherDerived>& other, - const RealScalar& prec) const { - return internal::isApprox_selector<Derived, OtherDerived>::run(derived(), other.derived(), prec); -} - -/** \returns \c true if the norm of \c *this is much smaller than \a other, - * within the precision determined by \a prec. - * - * \note The fuzzy compares are done multiplicatively. A vector \f$ v \f$ is - * considered to be much smaller than \f$ x \f$ within precision \f$ p \f$ if - * \f[ \Vert v \Vert \leqslant p\,\vert x\vert. \f] - * - * For matrices, the comparison is done using the Hilbert-Schmidt norm. For this reason, - * the value of the reference scalar \a other should come from the Hilbert-Schmidt norm - * of a reference matrix of same dimensions. - * - * \sa isApprox(), isMuchSmallerThan(const DenseBase<OtherDerived>&, RealScalar) const - */ -template <typename Derived> -EIGEN_DEVICE_FUNC constexpr bool DenseBase<Derived>::isMuchSmallerThan(const typename NumTraits<Scalar>::Real& other, - const RealScalar& prec) const { - return internal::isMuchSmallerThan_scalar_selector<Derived>::run(derived(), other, prec); -} - -/** \returns \c true if the norm of \c *this is much smaller than the norm of \a other, - * within the precision determined by \a prec. - * - * \note The fuzzy compares are done multiplicatively. A vector \f$ v \f$ is - * considered to be much smaller than a vector \f$ w \f$ within precision \f$ p \f$ if - * \f[ \Vert v \Vert \leqslant p\,\Vert w\Vert. \f] - * For matrices, the comparison is done using the Hilbert-Schmidt norm. - * - * \sa isApprox(), isMuchSmallerThan(const RealScalar&, RealScalar) const - */ -template <typename Derived> -template <typename OtherDerived> -EIGEN_DEVICE_FUNC constexpr bool DenseBase<Derived>::isMuchSmallerThan(const DenseBase<OtherDerived>& other, - const RealScalar& prec) const { - return internal::isMuchSmallerThan_object_selector<Derived, OtherDerived>::run(derived(), other.derived(), prec); -} - -} // end namespace Eigen - -#endif // EIGEN_FUZZY_H
diff --git a/Eigen/src/Core/GeneralProduct.h b/Eigen/src/Core/GeneralProduct.h index cc3a0ed..ac16c37 100644 --- a/Eigen/src/Core/GeneralProduct.h +++ b/Eigen/src/Core/GeneralProduct.h
@@ -25,7 +25,7 @@ // products/GeneralMatrixMatrix.h for more details. // The crossover belongs to the kernel the GEMM path would select, not to the // build: a scalar type that falls back to the generic gebp kernel keeps this -// value even in an SME build. The SME kernels have their own, larger crossovers +// value even in an SME build. The SME kernels have their own crossovers // -- sme_gemm_to_coeffbased_threshold and // EIGEN_SME_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD below. #ifndef EIGEN_GEMM_TO_COEFFBASED_THRESHOLD @@ -36,20 +36,17 @@ // Fixed-size products can reach the GEMM product path even when the // coeff-based evaluator is still faster. Keep this threshold separate so // runtime-size dispatch preserves the historical heuristic above. This default -// was tuned on the same Haswell system as the runtime threshold, and deliberately -// tracks EIGEN_GEMM_TO_COEFFBASED_THRESHOLD unless specialized independently. +// was tuned on the same Haswell system as the runtime threshold. It does not +// follow a user-defined EIGEN_GEMM_TO_COEFFBASED_THRESHOLD, which still applies +// at run time, so a fixed-size product's crossover is the larger of the two. #ifndef EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD -#define EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD (2 * EIGEN_GEMM_TO_COEFFBASED_THRESHOLD) +#define EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD 40 #endif #ifdef EIGEN_VECTORIZE_SME -// Kept as a separate knob for the scalar pairs the SME kernel claims. With the -// NEON small-block path those products run NEON packers and kernel below the -// crossover, so the generic fixed-size value applies; int, half, bfloat16, -// mixed real x complex, and double / complex<double> without FEAT_SME_F64F64 -// run the generic kernel and never read this one. +// Read only for SME-kernel scalar pairs; fitted on Apple M4 Pro to today's NEON small-block bounds (re-measure: #3194). #ifndef EIGEN_SME_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD -#define EIGEN_SME_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD +#define EIGEN_SME_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD 61 #endif #endif
diff --git a/Eigen/src/Core/GenericPacketMath.h b/Eigen/src/Core/GenericPacketMath.h index f4e4e6d..fbbec2b 100644 --- a/Eigen/src/Core/GenericPacketMath.h +++ b/Eigen/src/Core/GenericPacketMath.h
@@ -1189,6 +1189,10 @@ * Special math functions ***************************/ +// Declared ahead of pisnan_impl so the complex case can use a backend's pisnan for the real packet. +template <typename Packet> +EIGEN_DEVICE_FUNC inline Packet pisnan(const Packet& a); + // Implemented without ptrue: an all-ones float packet is a NaN bit pattern, which under // fast-math flags clang turns into a poison constant that deletes any expression it flows // into. @@ -1213,7 +1217,7 @@ static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { using RealPacket = typename unpacket_traits<Packet>::as_real; // A NaN in either the real or the imaginary lane marks the whole complex element. - Packet nan_lanes = Packet(pcmp_lt_or_nan<RealPacket>(a.v, a.v)); + Packet nan_lanes = Packet(pisnan<RealPacket>(a.v)); return por(nan_lanes, pcplxflip(nan_lanes)); } }; @@ -1893,7 +1897,7 @@ } /** \internal \returns a packet populated with values in the range [begin, begin + count). Elements - * outside this range are not defined. \a *from does not need to be aligned, and can be null if \a count is zero.*/ + * outside this range are zero. \a *from does not need to be aligned, and can be null if \a count is zero.*/ template <typename Packet> EIGEN_DEVICE_FUNC inline Packet ploaduSegment(const typename unpacket_traits<Packet>::type* from, Index begin, Index count) { @@ -1908,7 +1912,7 @@ } /** \internal \returns a packet populated with values in the range [begin, begin + count). Elements - * outside this range are not defined. \a *from must be aligned, and cannot be null.*/ + * outside this range are zero. \a *from must be aligned, and cannot be null.*/ template <typename Packet> EIGEN_DEVICE_FUNC inline Packet ploadSegment(const typename unpacket_traits<Packet>::type* from, Index begin, Index count) { @@ -1938,7 +1942,7 @@ } /** \internal \returns a packet populated with values in the range [begin, begin + count). Elements - * outside this range are not defined.*/ + * outside this range are zero.*/ template <typename Packet, int Alignment> EIGEN_DEVICE_FUNC inline Packet ploadtSegment(const typename unpacket_traits<Packet>::type* from, Index begin, Index count) {
diff --git a/Eigen/src/Core/IndexedView.h b/Eigen/src/Core/IndexedView.h index 0c71556..dab89b7 100644 --- a/Eigen/src/Core/IndexedView.h +++ b/Eigen/src/Core/IndexedView.h
@@ -216,7 +216,11 @@ Index outerIncrement() const { return traits<Derived>::IsRowMajor ? rowIncrement() : colIncrement(); } - std::decay_t<typename XprType::Scalar>* data() { + using ScalarWithConstIfNotLvalue = + std::conditional_t<is_lvalue<XprType>::value, std::decay_t<typename XprType::Scalar>, + const std::decay_t<typename XprType::Scalar>>; + + ScalarWithConstIfNotLvalue* data() { Index row_offset = this->rowIndices()[0] * this->nestedExpression().rowStride(); Index col_offset = this->colIndices()[0] * this->nestedExpression().colStride(); return this->nestedExpression().data() + row_offset + col_offset; @@ -232,14 +236,17 @@ EIGEN_IF_CONSTEXPR (traits<Derived>::InnerStrideAtCompileTime != Dynamic) { return traits<Derived>::InnerStrideAtCompileTime; } - return innerIncrement() * this->nestedExpression().innerStride(); + // A vector-shaped view need not share the nested storage order, so step along the view's own inner dimension. + return traits<Derived>::IsRowMajor ? colIncrement() * this->nestedExpression().colStride() + : rowIncrement() * this->nestedExpression().rowStride(); } EIGEN_DEVICE_FUNC constexpr Index outerStride() const noexcept { EIGEN_IF_CONSTEXPR (traits<Derived>::OuterStrideAtCompileTime != Dynamic) { return traits<Derived>::OuterStrideAtCompileTime; } - return outerIncrement() * this->nestedExpression().outerStride(); + return traits<Derived>::IsRowMajor ? rowIncrement() * this->nestedExpression().rowStride() + : colIncrement() * this->nestedExpression().colStride(); } };
diff --git a/Eigen/src/Core/InnerProduct.h b/Eigen/src/Core/InnerProduct.h index 6ce1edb..6957564 100644 --- a/Eigen/src/Core/InnerProduct.h +++ b/Eigen/src/Core/InnerProduct.h
@@ -211,7 +211,8 @@ static constexpr bool PacketAccess = packet_traits<Scalar>::HasMul && packet_traits<Scalar>::HasAdd; }; -template <typename Lhs, typename Rhs, bool Conj> +// Backends specialize Enable = enable_if_t<...> (void); Enable = false_type always names this generic definition. +template <typename Lhs, typename Rhs, bool Conj, typename Enable = void> struct default_inner_product_impl { using LhsScalar = typename traits<Lhs>::Scalar; using RhsScalar = typename traits<Rhs>::Scalar;
diff --git a/Eigen/src/Core/MathFunctions.h b/Eigen/src/Core/MathFunctions.h index 879dff2..8479cc9 100644 --- a/Eigen/src/Core/MathFunctions.h +++ b/Eigen/src/Core/MathFunctions.h
@@ -232,6 +232,16 @@ } }; +// MSVC's STL evaluates T * complex<T> as complex<T>(a) * b, whose 0 * inf terms turn 2 * (inf, 2) into +// (inf, NaN). Scale component-wise, as complex<T> * T, libstdc++, libc++, and the GPU overloads do. +template <typename RealScalar> +struct mul_impl<RealScalar, std::complex<RealScalar>, false> { + EIGEN_DEVICE_FUNC static constexpr EIGEN_ALWAYS_INLINE std::complex<RealScalar> run( + const RealScalar& a, const std::complex<RealScalar>& b) { + return std::complex<RealScalar>(a * b.real(), a * b.imag()); + } +}; + /** \internal \returns a * b, wrapping rather than overflowing the signed type that integral promotion * would give narrow unsigned operands. The result keeps operator*'s type, which for an expression-template * Scalar is a lazy expression: consume it rather than binding it to auto. */ @@ -1055,6 +1065,29 @@ return magnitude == 0; } +// Floating-point comparisons can treat subnormals as zero under DAZ/FTZ. +template <typename Scalar> +struct sign_impl_binary { + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Scalar& a) { + using Binary = binary_floating_point_traits<Scalar>; + using Bits = typename Binary::Bits; + const Bits bits = Binary::bits(a); + const Bits magnitude = bits & ~Binary::kSignBit; + // magnitude | -magnitude has its high bit set iff magnitude is nonzero. + const Bits nonzero = (magnitude | (Bits(0) - magnitude)) >> (sizeof(Bits) * CHAR_BIT - 1); + const Bits nonzero_mask = Bits(0) - nonzero; + const Bits nan_mask = Bits(0) - Bits(magnitude > Binary::kExponentMask); + const Bits signed_one = (bits & Binary::kSignBit) | Binary::bits(Scalar(1)); + return numext::bit_cast<Scalar>((bits & nan_mask) | (signed_one & nonzero_mask & ~nan_mask)); + } +}; + +template <> +struct sign_impl<float, false, false> : sign_impl_binary<float> {}; + +template <> +struct sign_impl<double, false, false> : sign_impl_binary<double> {}; + template <typename Scalar> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_exactly_zero_no_flush_impl(const Scalar& value, true_type) { return is_zero_magnitude_bits<Scalar>(binary_floating_point_traits<Scalar>::magnitude(value));
diff --git a/Eigen/src/Core/MathFunctionsImpl.h b/Eigen/src/Core/MathFunctionsImpl.h index 6296aff..de498a1 100644 --- a/Eigen/src/Core/MathFunctionsImpl.h +++ b/Eigen/src/Core/MathFunctionsImpl.h
@@ -171,7 +171,7 @@ }; template <typename ComplexT, bool Reciprocal> -EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE constexpr ComplexT complex_sqrt_extreme(const ComplexT& z) { +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE ComplexT complex_sqrt_extreme(const ComplexT& z) { using T = typename NumTraits<ComplexT>::Real; const T x = numext::real(z); const T y = numext::imag(z);
diff --git a/Eigen/src/Core/Matrix.h b/Eigen/src/Core/Matrix.h index b4a51fc..1b39778 100644 --- a/Eigen/src/Core/Matrix.h +++ b/Eigen/src/Core/Matrix.h
@@ -247,6 +247,11 @@ * For dynamic-size matrices, creates an empty matrix of size 0. Does not allocate any array. Such a matrix * is called a null matrix. An existing matrix can also be turned into a null matrix by resizing it to 0. * + * Unless \c EIGEN_INITIALIZE_MATRICES_BY_ZERO or \c EIGEN_INITIALIZE_MATRICES_BY_NAN is defined, this constructor is + * defaulted, so value-initialization (<tt>Matrix m{};</tt>, <tt>Matrix()</tt>, or <tt>m()</tt> in a member + * initializer list) zero-initializes the whole object first, including the unused part of a fixed-maximum-size + * inline buffer. Default-initialization (<tt>Matrix m;</tt>) only default-initializes the coefficients. + * * \sa resize(Index,Index) */ #if defined(EIGEN_INITIALIZE_COEFFS)
diff --git a/Eigen/src/Core/MatrixNorms.h b/Eigen/src/Core/MatrixNorms.h index b68a75c..e573826 100644 --- a/Eigen/src/Core/MatrixNorms.h +++ b/Eigen/src/Core/MatrixNorms.h
@@ -85,8 +85,9 @@ // The running sums of a pass over two columns. struct Pass { - RPacket acc0 = pzero(RPacket()); - RPacket acc1 = pzero(RPacket()); + // pset1 rather than pzero(RPacket()): nvc++ 26.1 crashes on a value-initialized __m512 (#3193). + RPacket acc0 = pset1<RPacket>(Real(0)); + RPacket acc1 = pset1<RPacket>(Real(0)); template <typename SumsEvaluator> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void step(const RPacket& u, const RPacket& v, SumsEvaluator& s, Index i) { RPacket a = pabs(u); @@ -125,8 +126,8 @@ } struct Pass { - RPacket acc = pzero(RPacket()); // |u| in the even lanes, |v| in the odd ones - RPacket peak_ = pzero(RPacket()); // the largest component seen + RPacket acc = pset1<RPacket>(Real(0)); // |u| in the even lanes, |v| in the odd ones + RPacket peak_ = pset1<RPacket>(Real(0)); // the largest component seen template <typename SumsEvaluator> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void step(const RPacket& u, const RPacket& v, SumsEvaluator& s, Index i) { peak_ = pmax(peak_, pmax(pabs(u), pabs(v)));
diff --git a/Eigen/src/Core/NumTraits.h b/Eigen/src/Core/NumTraits.h index 158e4f2..3be727a 100644 --- a/Eigen/src/Core/NumTraits.h +++ b/Eigen/src/Core/NumTraits.h
@@ -164,8 +164,8 @@ * called, and to 0 if it is safe not to call it. Default is 0 if \a T is an arithmetic type, and 1 otherwise. * \li An epsilon() function which, unlike <a href="http://en.cppreference.com/w/cpp/types/numeric_limits/epsilon"> * `std::numeric_limits::epsilon()`</a>, it returns a \c Real instead of a \a T. - * \li A dummy_precision() function returning a weak epsilon value. It is mainly used as a default value by the fuzzy - * comparison operators. + * \li A dummy_precision() function returning a default relative tolerance for approximate comparisons. + * This is distinct from epsilon() and does not describe the rounding error of an algorithm. * \li highest() and lowest() functions returning the highest and lowest possible values respectively. * \li digits() function returning the number of radix digits (non-sign digits for integers, mantissa for floating-point). * This is the analogue of <a href="http://en.cppreference.com/w/cpp/types/numeric_limits/digits">
diff --git a/Eigen/src/Core/PermutationMatrix.h b/Eigen/src/Core/PermutationMatrix.h index eff2cc2..cb073fd 100644 --- a/Eigen/src/Core/PermutationMatrix.h +++ b/Eigen/src/Core/PermutationMatrix.h
@@ -21,6 +21,47 @@ enum PermPermProduct_t { PermPermProduct }; +/** \internal + * Nullary functor reading a permutation as a dense matrix of Scalar: P has its ones at (indices(k), k), + * and its inverse (Transposed) at (k, indices(k)). It backs the lazy sums of a dense or diagonal matrix + * with a permutation, whose scalar type the permutation borrows. + */ +template <typename Scalar, typename IndicesType, bool Transposed> +struct permutation_dense_op { + EIGEN_DEVICE_FUNC explicit permutation_dense_op(const IndicesType& indices) : m_indices(indices) {} + + template <typename IndexType> + EIGEN_DEVICE_FUNC Scalar operator()(IndexType row, IndexType col) const { + const Index k = Transposed ? Index(row) : Index(col); + const Index image = Transposed ? Index(col) : Index(row); + return Index(m_indices.coeff(k)) == image ? Scalar(1) : Scalar(0); + } + + EIGEN_DEVICE_FUNC const remove_all_t<typename IndicesType::Nested>& indices() const { return m_indices; } + + typename IndicesType::Nested m_indices; +}; + +template <typename Scalar, typename IndicesType, bool Transposed> +struct functor_traits<permutation_dense_op<Scalar, IndicesType, Transposed>> { + static constexpr int Cost = int(NumTraits<typename IndicesType::Scalar>::ReadCost) + int(NumTraits<Scalar>::AddCost); + static constexpr bool PacketAccess = false; + static constexpr bool IsRepeatable = true; +}; + +template <typename Scalar, bool Transposed, typename PermutationType> +struct permutation_dense_expression { + using IndicesType = remove_all_t<typename PermutationType::IndicesType>; + using PlainObject = Matrix<Scalar, PermutationType::RowsAtCompileTime, PermutationType::ColsAtCompileTime, 0, + PermutationType::MaxRowsAtCompileTime, PermutationType::MaxColsAtCompileTime>; + using type = CwiseNullaryOp<permutation_dense_op<Scalar, IndicesType, Transposed>, PlainObject>; + + static EIGEN_DEVICE_FUNC type run(const PermutationType& permutation) { + return type(permutation.rows(), permutation.cols(), + permutation_dense_op<Scalar, IndicesType, Transposed>(permutation.indices())); + } +}; + } // end namespace internal /** \class PermutationBase @@ -191,6 +232,12 @@ * \note \blank \note_try_to_help_rvo */ inline InverseReturnType transpose() const { return InverseReturnType(derived()); } + /** \returns the adjoint of the permutation matrix. Its entries are real and it is orthogonal, so this equals + * transpose() and inverse(). + * + * \note \blank \note_try_to_help_rvo + */ + InverseReturnType adjoint() const { return InverseReturnType(derived()); } /**** multiplication helpers to hopefully get RVO ****/ @@ -496,6 +543,81 @@ return Product<PermutationDerived, MatrixDerived, DefaultProduct>(permutation.derived(), matrix.derived()); } +// Sums with a permutation are lazy dense expressions: the permutation is read as a 0/1 matrix with the scalar +// type of the other operand, so no dense copy of it is formed. + +/** \returns the lazy sum of the dense matrix \a matrix and the permutation matrix \a permutation */ +template <typename MatrixDerived, typename PermutationDerived> +EIGEN_DEVICE_FUNC auto operator+(const MatrixBase<MatrixDerived>& matrix, + const PermutationBase<PermutationDerived>& permutation) { + return matrix.derived() + + internal::permutation_dense_expression<typename MatrixDerived::Scalar, false, PermutationDerived>::run( + permutation.derived()); +} + +/** \returns the lazy sum of the permutation matrix \a permutation and the dense matrix \a matrix */ +template <typename PermutationDerived, typename MatrixDerived> +EIGEN_DEVICE_FUNC auto operator+(const PermutationBase<PermutationDerived>& permutation, + const MatrixBase<MatrixDerived>& matrix) { + return internal::permutation_dense_expression<typename MatrixDerived::Scalar, false, PermutationDerived>::run( + permutation.derived()) + + matrix.derived(); +} + +/** \returns the lazy difference of the dense matrix \a matrix and the permutation matrix \a permutation */ +template <typename MatrixDerived, typename PermutationDerived> +EIGEN_DEVICE_FUNC auto operator-(const MatrixBase<MatrixDerived>& matrix, + const PermutationBase<PermutationDerived>& permutation) { + return matrix.derived() - + internal::permutation_dense_expression<typename MatrixDerived::Scalar, false, PermutationDerived>::run( + permutation.derived()); +} + +/** \returns the lazy difference of the permutation matrix \a permutation and the dense matrix \a matrix */ +template <typename PermutationDerived, typename MatrixDerived> +EIGEN_DEVICE_FUNC auto operator-(const PermutationBase<PermutationDerived>& permutation, + const MatrixBase<MatrixDerived>& matrix) { + return internal::permutation_dense_expression<typename MatrixDerived::Scalar, false, PermutationDerived>::run( + permutation.derived()) - + matrix.derived(); +} + +/** \returns the lazy sum of the diagonal matrix \a diagonal and the permutation matrix \a permutation */ +template <typename DiagonalDerived, typename PermutationDerived> +EIGEN_DEVICE_FUNC auto operator+(const DiagonalBase<DiagonalDerived>& diagonal, + const PermutationBase<PermutationDerived>& permutation) { + return diagonal.derived() + + internal::permutation_dense_expression<typename DiagonalDerived::Scalar, false, PermutationDerived>::run( + permutation.derived()); +} + +/** \returns the lazy sum of the permutation matrix \a permutation and the diagonal matrix \a diagonal */ +template <typename PermutationDerived, typename DiagonalDerived> +EIGEN_DEVICE_FUNC auto operator+(const PermutationBase<PermutationDerived>& permutation, + const DiagonalBase<DiagonalDerived>& diagonal) { + return internal::permutation_dense_expression<typename DiagonalDerived::Scalar, false, PermutationDerived>::run( + permutation.derived()) + + diagonal.derived(); +} + +/** \returns the lazy difference of the diagonal matrix \a diagonal and the permutation matrix \a permutation */ +template <typename DiagonalDerived, typename PermutationDerived> +EIGEN_DEVICE_FUNC auto operator-(const DiagonalBase<DiagonalDerived>& diagonal, + const PermutationBase<PermutationDerived>& permutation) { + return diagonal.derived() - + internal::permutation_dense_expression<typename DiagonalDerived::Scalar, false, PermutationDerived>::run( + permutation.derived()); +} + +/** \returns the lazy difference of the permutation matrix \a permutation and the diagonal matrix \a diagonal */ +template <typename PermutationDerived, typename DiagonalDerived> +EIGEN_DEVICE_FUNC auto operator-(const PermutationBase<PermutationDerived>& permutation, + const DiagonalBase<DiagonalDerived>& diagonal) { + return internal::permutation_dense_expression<typename DiagonalDerived::Scalar, false, PermutationDerived>::run( + permutation.derived()) - + diagonal.derived(); +} + template <typename PermutationType> class InverseImpl<PermutationType, PermutationStorage> : public EigenBase<Inverse<PermutationType> > { using PlainPermutationType = typename PermutationType::PlainPermutationType; @@ -546,6 +668,56 @@ const Product<InverseType, OtherDerived, DefaultProduct> operator*(const MatrixBase<OtherDerived>& matrix) const { return Product<InverseType, OtherDerived, DefaultProduct>(derived(), matrix.derived()); } + + // Lazy sums with a dense or diagonal matrix, as for PermutationBase; the inverse is read as the transposed 0/1 + // matrix. + /** \returns the lazy sum of the inverse permutation and the dense matrix \a matrix */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC auto operator+(const MatrixBase<OtherDerived>& matrix) const { + return denseExpression<typename OtherDerived::Scalar>() + matrix.derived(); + } + /** \returns the lazy sum of the dense matrix \a matrix and the inverse permutation \a inverse */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC friend auto operator+(const MatrixBase<OtherDerived>& matrix, const InverseType& inverse) { + return matrix.derived() + inverse.template denseExpression<typename OtherDerived::Scalar>(); + } + /** \returns the lazy difference of the inverse permutation and the dense matrix \a matrix */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC auto operator-(const MatrixBase<OtherDerived>& matrix) const { + return denseExpression<typename OtherDerived::Scalar>() - matrix.derived(); + } + /** \returns the lazy difference of the dense matrix \a matrix and the inverse permutation \a inverse */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC friend auto operator-(const MatrixBase<OtherDerived>& matrix, const InverseType& inverse) { + return matrix.derived() - inverse.template denseExpression<typename OtherDerived::Scalar>(); + } + /** \returns the lazy sum of the inverse permutation and the diagonal matrix \a diagonal */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC auto operator+(const DiagonalBase<OtherDerived>& diagonal) const { + return denseExpression<typename OtherDerived::Scalar>() + diagonal.derived(); + } + /** \returns the lazy sum of the diagonal matrix \a diagonal and the inverse permutation \a inverse */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC friend auto operator+(const DiagonalBase<OtherDerived>& diagonal, const InverseType& inverse) { + return diagonal.derived() + inverse.template denseExpression<typename OtherDerived::Scalar>(); + } + /** \returns the lazy difference of the inverse permutation and the diagonal matrix \a diagonal */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC auto operator-(const DiagonalBase<OtherDerived>& diagonal) const { + return denseExpression<typename OtherDerived::Scalar>() - diagonal.derived(); + } + /** \returns the lazy difference of the diagonal matrix \a diagonal and the inverse permutation \a inverse */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC friend auto operator-(const DiagonalBase<OtherDerived>& diagonal, const InverseType& inverse) { + return diagonal.derived() - inverse.template denseExpression<typename OtherDerived::Scalar>(); + } + + private: + template <typename Scalar> + EIGEN_DEVICE_FUNC auto denseExpression() const { + using Nested = internal::remove_all_t<typename InverseType::XprTypeNestedCleaned>; + return internal::permutation_dense_expression<Scalar, true, Nested>::run(derived().nestedExpression()); + } }; template <typename Derived> @@ -560,6 +732,127 @@ using Kind = EigenBase2EigenBase; }; +// Dense ?= (dense or diagonal) +/- permutation, in either order. The permutation's one in column k is at row +// indices(k), and at column indices(k) of row k for its transpose; the column kernels below need one nonzero per +// destination column, so P pairs with a column-major and P^T with a row-major destination, and the other pairings +// stay on the generic path. +template <typename Scalar, typename IndicesType> +struct permutation_column_nonzeros { + EIGEN_DEVICE_FUNC explicit permutation_column_nonzeros(const IndicesType& indices) : m_indices(indices) {} + EIGEN_DEVICE_FUNC Index row(Index k) const { return Index(m_indices.coeff(k)); } + EIGEN_DEVICE_FUNC Scalar value(Index) const { return Scalar(1); } + const IndicesType& m_indices; +}; + +template <typename Scalar, typename IndicesType, bool Transposed, typename Plain> +using permutation_dense_xpr = CwiseNullaryOp<permutation_dense_op<Scalar, IndicesType, Transposed>, Plain>; + +template <typename Scalar, typename IndicesType, bool Transposed, typename Plain> +struct is_permutation_dense_xpr<permutation_dense_xpr<Scalar, IndicesType, Transposed, Plain>> : std::true_type {}; + +// A diagonal operand never goes through the block pass, so only a dense one is subject to +// dense_block_pass_cannot_overflow. +template <typename Dst, typename OtherXpr, bool Transposed, typename Functor, bool NegateOther> +struct dense_permutation_sum_fast_path + : bool_constant<(is_dense_shape<OtherXpr>::value || is_diagonal_shape<OtherXpr>::value) && + !is_permutation_dense_xpr<OtherXpr>::value && bool(Dst::IsRowMajor) == Transposed && + additive_assign_sign<Functor>::value != 0 && + std::is_same<typename Dst::Scalar, typename OtherXpr::Scalar>::value && + (is_diagonal_shape<OtherXpr>::value || + dense_block_pass_cannot_overflow<typename Dst::Scalar, Functor, NegateOther>::value)> {}; + +// dst ?= op(lhs, rhs) for a diagonal d and a permutation whose one in column k is at row r: op(d(k), [r == k]) at +// (k, k), op(0, 1) or op(1, 0) at (r, k) when r != k, and structural zeros, which only = writes. A block's d(k) are +// read before the block is written. +template <bool DiagonalOnLeft> +struct diagonal_permutation_sum_assignment { + template <typename Dst, typename Diagonal, typename Permutation, typename BinaryOp, typename Functor> + EIGEN_DEVICE_FUNC static void run(Dst& dst, const Diagonal& diagonal, const Permutation& permutation, + const BinaryOp& op, const Functor& func) { + using Scalar = typename Dst::Scalar; + constexpr Index kBlockColumns = 32; + const Scalar offDiagonalOne = DiagonalOnLeft ? op(Scalar(0), Scalar(1)) : op(Scalar(1), Scalar(0)); + Scalar atDiagonal[kBlockColumns]; + for (Index j = 0; j < dst.cols(); j += kBlockColumns) { + const Index columns = numext::mini(Index(kBlockColumns), dst.cols() - j); + for (Index k = 0; k < columns; ++k) { + const Scalar d = diagonal.value(j + k); + const Scalar p = permutation.row(j + k) == j + k ? Scalar(1) : Scalar(0); + atDiagonal[k] = DiagonalOnLeft ? op(d, p) : op(p, d); + } + EIGEN_IF_CONSTEXPR (is_plain_assign<Functor>::value) { + dst.middleCols(j, columns).setZero(); + } + for (Index k = 0; k < columns; ++k) { + const Index r = permutation.row(j + k); + func.assignCoeff(dst.coeffRef(j + k, j + k), atDiagonal[k]); + if (r != j + k) { + func.assignCoeff(dst.coeffRef(r, j + k), offDiagonalOne); + } + } + } + } +}; + +template <bool OtherIsDiagonal, bool OtherOnLeft> +struct permutation_sum_assignment { + template <typename Dst, typename SrcXprType, typename OtherXpr, typename Nonzeros, typename Functor> + EIGEN_DEVICE_FUNC static void run(Dst& dst, const SrcXprType& src, const OtherXpr& other, const Nonzeros& ones, + const Functor& func) { + dense_structured_sum_assignment<OtherOnLeft>::assign(dst, src, other, ones, func); + } +}; + +template <bool OtherOnLeft> +struct permutation_sum_assignment<true, OtherOnLeft> { + template <typename Dst, typename SrcXprType, typename OtherXpr, typename Nonzeros, typename Functor> + EIGEN_DEVICE_FUNC static void run(Dst& dst, const SrcXprType& src, const OtherXpr& other, const Nonzeros& ones, + const Functor& func) { + const diagonal_column_nonzeros<OtherXpr> diagonal(other); + resize_if_allowed(dst, src, func); + auto&& dstView = column_major_view<bool(Dst::IsRowMajor)>::run(dst); + diagonal_permutation_sum_assignment<OtherOnLeft>::run(dstView, diagonal, ones, src.functor(), func); + } +}; + +// other +/- permutation +template <typename DstXprType, typename BinaryOp, typename OtherXpr, typename Scalar, typename IndicesType, + bool Transposed, typename Plain, typename Functor> +struct Assignment< + DstXprType, + CwiseBinaryOp<BinaryOp, const OtherXpr, const permutation_dense_xpr<Scalar, IndicesType, Transposed, Plain>>, + Functor, Dense2Dense, + std::enable_if_t<is_additive_binary_op<BinaryOp>::value && + dense_permutation_sum_fast_path<DstXprType, OtherXpr, Transposed, Functor, false>::value>> { + using SrcXprType = + CwiseBinaryOp<BinaryOp, const OtherXpr, const permutation_dense_xpr<Scalar, IndicesType, Transposed, Plain>>; + EIGEN_DEVICE_FUNC static void run(DstXprType& dst, const SrcXprType& src, const Functor& func) { + const auto& indices = src.rhs().functor().indices(); + permutation_sum_assignment<is_diagonal_shape<OtherXpr>::value, true>::run( + dst, src, src.lhs(), permutation_column_nonzeros<Scalar, remove_all_t<decltype(indices)>>(indices), func); + } +}; + +// permutation +/- other; a dense product on the right is left to the "xpr + product" rule. +template <typename DstXprType, typename BinaryOp, typename OtherXpr, typename Scalar, typename IndicesType, + bool Transposed, typename Plain, typename Functor> +struct Assignment< + DstXprType, + CwiseBinaryOp<BinaryOp, const permutation_dense_xpr<Scalar, IndicesType, Transposed, Plain>, const OtherXpr>, + Functor, Dense2Dense, + std::enable_if_t<is_additive_binary_op<BinaryOp>::value && + dense_permutation_sum_fast_path<DstXprType, OtherXpr, Transposed, Functor, + is_difference_op<BinaryOp>::value>::value && + !is_default_product<OtherXpr>::value>> { + using SrcXprType = + CwiseBinaryOp<BinaryOp, const permutation_dense_xpr<Scalar, IndicesType, Transposed, Plain>, const OtherXpr>; + EIGEN_DEVICE_FUNC static void run(DstXprType& dst, const SrcXprType& src, const Functor& func) { + const auto& indices = src.lhs().functor().indices(); + permutation_sum_assignment<is_diagonal_shape<OtherXpr>::value, false>::run( + dst, src, src.rhs(), permutation_column_nonzeros<Scalar, remove_all_t<decltype(indices)>>(indices), func); + } +}; + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/PlainObjectBase.h b/Eigen/src/Core/PlainObjectBase.h index a255d3e..6c92f6b 100644 --- a/Eigen/src/Core/PlainObjectBase.h +++ b/Eigen/src/Core/PlainObjectBase.h
@@ -485,8 +485,7 @@ */ template <typename... ArgTypes> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PlainObjectBase(const Scalar& a0, const Scalar& a1, const Scalar& a2, - const Scalar& a3, const ArgTypes&... args) - : m_storage() { + const Scalar& a3, const ArgTypes&... args) { EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(PlainObjectBase, sizeof...(args) + 4); m_storage.data()[0] = a0; m_storage.data()[1] = a1; @@ -501,8 +500,7 @@ * lists */ EIGEN_DEVICE_FUNC explicit constexpr EIGEN_STRONG_INLINE PlainObjectBase( - const std::initializer_list<std::initializer_list<Scalar>>& list) - : m_storage() { + const std::initializer_list<std::initializer_list<Scalar>>& list) { size_t list_size = 0; if (list.begin() != list.end()) { list_size = list.begin()->size(); @@ -541,14 +539,14 @@ /** \sa PlainObjectBase::operator=(const EigenBase<OtherDerived>&) */ template <typename OtherDerived> - EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE PlainObjectBase(const DenseBase<OtherDerived>& other) : m_storage() { + EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE PlainObjectBase(const DenseBase<OtherDerived>& other) { resizeLike(other); _set_noalias(other); } /** \sa PlainObjectBase::operator=(const EigenBase<OtherDerived>&) */ template <typename OtherDerived> - EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE PlainObjectBase(const EigenBase<OtherDerived>& other) : m_storage() { + EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE PlainObjectBase(const EigenBase<OtherDerived>& other) { resizeLike(other); *this = other.derived(); } @@ -705,8 +703,9 @@ template <typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _resize_to_match(const EigenBase<OtherDerived>& other) { #ifdef EIGEN_NO_AUTOMATIC_RESIZING - eigen_assert((this->size() == 0 || (IsVectorAtCompileTime ? (this->size() == other.size()) - : (rows() == other.rows() && cols() == other.cols()))) && + eigen_assert((this->size() == 0 || + (IsVectorAtCompileTime ? ((other.rows() == 1 || other.cols() == 1) && this->size() == other.size()) + : (rows() == other.rows() && cols() == other.cols()))) && "Size mismatch. Automatic resizing is disabled because EIGEN_NO_AUTOMATIC_RESIZING is defined"); if (this->size() == 0) resizeLike(other); EIGEN_ONLY_USED_FOR_DEBUG(other);
diff --git a/Eigen/src/Core/ProductEvaluators.h b/Eigen/src/Core/ProductEvaluators.h index bc32109..3166947 100644 --- a/Eigen/src/Core/ProductEvaluators.h +++ b/Eigen/src/Core/ProductEvaluators.h
@@ -165,10 +165,8 @@ std::enable_if_t<(Options == DefaultProduct || Options == AliasFreeProduct)>> { using SrcXprType = Product<Lhs, Rhs, Options>; static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src, - const internal::assign_op<Scalar, Scalar>&) { - Index dstRows = src.rows(); - Index dstCols = src.cols(); - if ((dst.rows() != dstRows) || (dst.cols() != dstCols)) dst.resize(dstRows, dstCols); + const internal::assign_op<Scalar, Scalar>& func) { + resize_if_allowed(dst, src, func); // FIXME shall we handle nested_eval here? generic_product_impl<Lhs, Rhs>::evalTo(dst, src.lhs(), src.rhs()); } @@ -320,7 +318,7 @@ return dst.rows() <= 16 && dst.cols() <= 16; } -template <typename Dst, typename Lhs, typename Rhs, typename Func, typename Scalar> +template <bool ApplyScale = true, typename Dst, typename Lhs, typename Rhs, typename Func, typename Scalar> void EIGEN_DEVICE_FUNC outer_product_selector_run_small(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Func& func, const Scalar& alpha, const std::false_type&) { evaluator<Rhs> rhsEval(rhs); @@ -328,14 +326,18 @@ const Index rows = dst.rows(); const Index cols = dst.cols(); for (Index j = 0; j < cols; ++j) { - const Scalar rhs_j = rhsEval.coeff(Index(0), j); + const typename Rhs::Scalar rhs_j = rhsEval.coeff(Index(0), j); for (Index i = 0; i < rows; ++i) { - func.assignCoeff(dst.coeffRef(i, j), alpha * (rhs_j * actual_lhs.coeff(i, Index(0)))); + EIGEN_IF_CONSTEXPR (ApplyScale) { + func.assignCoeff(dst.coeffRef(i, j), internal::mul(alpha, internal::mul(rhs_j, actual_lhs.coeff(i, Index(0))))); + } else { + func.assignCoeff(dst.coeffRef(i, j), internal::mul(rhs_j, actual_lhs.coeff(i, Index(0)))); + } } } } -template <typename Dst, typename Lhs, typename Rhs, typename Func, typename Scalar> +template <bool ApplyScale = true, typename Dst, typename Lhs, typename Rhs, typename Func, typename Scalar> void EIGEN_DEVICE_FUNC outer_product_selector_run_small(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Func& func, const Scalar& alpha, const std::true_type&) { evaluator<Lhs> lhsEval(lhs); @@ -343,9 +345,13 @@ const Index rows = dst.rows(); const Index cols = dst.cols(); for (Index i = 0; i < rows; ++i) { - const Scalar lhs_i = lhsEval.coeff(i, Index(0)); + const typename Lhs::Scalar lhs_i = lhsEval.coeff(i, Index(0)); for (Index j = 0; j < cols; ++j) { - func.assignCoeff(dst.coeffRef(i, j), alpha * (lhs_i * actual_rhs.coeff(Index(0), j))); + EIGEN_IF_CONSTEXPR (ApplyScale) { + func.assignCoeff(dst.coeffRef(i, j), internal::mul(alpha, internal::mul(lhs_i, actual_rhs.coeff(Index(0), j)))); + } else { + func.assignCoeff(dst.coeffRef(i, j), internal::mul(lhs_i, actual_rhs.coeff(Index(0), j))); + } } } } @@ -391,8 +397,8 @@ template <typename Dst> static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { if (internal::outer_product_use_small_assignment(dst)) { - internal::outer_product_selector_run_small(dst, lhs, rhs, internal::assign_op<typename Dst::Scalar, Scalar>(), - Scalar(1), is_row_major<Dst>()); + internal::outer_product_selector_run_small<false>( + dst, lhs, rhs, internal::assign_op<typename Dst::Scalar, Scalar>(), Scalar(1), is_row_major<Dst>()); } else { internal::outer_product_selector_run(dst, lhs, rhs, set(), is_row_major<Dst>()); } @@ -401,8 +407,8 @@ template <typename Dst> static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { if (internal::outer_product_use_small_assignment(dst)) { - internal::outer_product_selector_run_small(dst, lhs, rhs, internal::add_assign_op<typename Dst::Scalar, Scalar>(), - Scalar(1), is_row_major<Dst>()); + internal::outer_product_selector_run_small<false>( + dst, lhs, rhs, internal::add_assign_op<typename Dst::Scalar, Scalar>(), Scalar(1), is_row_major<Dst>()); } else { internal::outer_product_selector_run(dst, lhs, rhs, add(), is_row_major<Dst>()); } @@ -411,8 +417,8 @@ template <typename Dst> static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { if (internal::outer_product_use_small_assignment(dst)) { - internal::outer_product_selector_run_small(dst, lhs, rhs, internal::sub_assign_op<typename Dst::Scalar, Scalar>(), - Scalar(1), is_row_major<Dst>()); + internal::outer_product_selector_run_small<false>( + dst, lhs, rhs, internal::sub_assign_op<typename Dst::Scalar, Scalar>(), Scalar(1), is_row_major<Dst>()); } else { internal::outer_product_selector_run(dst, lhs, rhs, sub(), is_row_major<Dst>()); } @@ -570,18 +576,20 @@ // Packet2cd/Packet4cf tail is a sizeable win there too. // // The destination must also store the vectorized axis contiguously -// (InnerStrideAtCompileTime == 1): the cascade writes whole packets with a -// single contiguous store, so a strided destination such as `m.row(i)` of a -// column-major matrix would clobber neighbouring coefficients. The generic -// assignment path makes this same packet-access check on the destination; here -// we must replicate it since we bypass that path. -template <typename ProdEval, typename Dst> +// (DirectAccessBit and InnerStrideAtCompileTime == 1): the cascade writes whole +// packets with a single unaligned store, so a strided destination such as +// `m.row(i)` of a column-major matrix would clobber neighbouring coefficients. +// The generic assignment path makes this check along with MightVectorize and +// EIGEN_UNALIGNED_VECTORIZE; here we must replicate them since we bypass that path. +template <typename ProdEval, typename Dst, typename Func> struct product_packet_cascade_traits { using Scalar = typename ProdEval::Scalar; + using DstEval = evaluator<Dst>; static constexpr int Rows = ProdEval::RowsAtCompileTime; static constexpr int Cols = ProdEval::ColsAtCompileTime; static constexpr bool DstRowMajor = bool(Dst::IsRowMajor); - static constexpr bool DstContiguous = int(Dst::InnerStrideAtCompileTime) == 1; + static constexpr bool DstContiguous = + (int(DstEval::Flags) & DirectAccessBit) != 0 && int(Dst::InnerStrideAtCompileTime) == 1; // Orientation of the packets the product evaluator natively produces. When a // product can vectorize *both* the lhs (columns) and the rhs (rows), its // packet() returns column packets (EvalToRowMajor == 0). The cascade axis must @@ -596,11 +604,15 @@ using Packet = typename find_largest_packet<Scalar, Extent>::type; static constexpr int LargestSize = unpacket_traits<Packet>::size; static constexpr int GenericSize = largest_exact_divisor_size<typename packet_traits<Scalar>::type, Extent>::value; + static constexpr bool MightVectorize = + bool(EIGEN_UNALIGNED_VECTORIZE) && + (int(DstEval::Flags) & int(ProdEval::Flags) & int(ActualPacketAccessBit)) != 0 && + bool(functor_traits<Func>::PacketAccess); // SameType mirrors the product's own PacketAccessBit (SameType && (CanVectorizeLhs || // CanVectorizeRhs)): CanVectorizeLhs/Rhs alone test only the operand's packet access, so a // mixed-scalar product (e.g. double * complex) would otherwise enable the cascade and try to // read the wrong-width packet out of an operand. Mixed types keep the generic (scalar) path. - static constexpr bool Enable = bool(ProdEval::SameType) && (VecLhs || VecRhs) && DstContiguous && + static constexpr bool Enable = bool(ProdEval::SameType) && (VecLhs || VecRhs) && DstContiguous && MightVectorize && (LargestSize <= Extent) && (LargestSize > GenericSize); }; @@ -627,7 +639,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void product_packet_assign(std::true_type, const Func& func, Dst& dst, const Lhs& lhs, const Rhs& rhs) { using ProdEval = product_evaluator<Product<Lhs, Rhs, LazyProduct>, CoeffBasedProductMode, DenseShape, DenseShape>; - using Traits = product_packet_cascade_traits<ProdEval, Dst>; + using Traits = product_packet_cascade_traits<ProdEval, Dst, Func>; const ProdEval prodEval(lhs, rhs); product_run_packet_cascade<Traits>(func, dst, prodEval); } @@ -658,18 +670,18 @@ static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { // dst.noalias() += lhs.lazyProduct(rhs); using ProdEval = product_evaluator<Product<Lhs, Rhs, LazyProduct>, CoeffBasedProductMode, DenseShape, DenseShape>; - using Traits = product_packet_cascade_traits<ProdEval, Dst>; - product_packet_assign(bool_constant<Traits::Enable>(), internal::add_assign_op<typename Dst::Scalar, Scalar>(), dst, - lhs, rhs); + using Func = internal::add_assign_op<typename Dst::Scalar, Scalar>; + using Traits = product_packet_cascade_traits<ProdEval, Dst, Func>; + product_packet_assign(bool_constant<Traits::Enable>(), Func(), dst, lhs, rhs); } template <typename Dst> static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { // dst.noalias() -= lhs.lazyProduct(rhs); using ProdEval = product_evaluator<Product<Lhs, Rhs, LazyProduct>, CoeffBasedProductMode, DenseShape, DenseShape>; - using Traits = product_packet_cascade_traits<ProdEval, Dst>; - product_packet_assign(bool_constant<Traits::Enable>(), internal::sub_assign_op<typename Dst::Scalar, Scalar>(), dst, - lhs, rhs); + using Func = internal::sub_assign_op<typename Dst::Scalar, Scalar>; + using Traits = product_packet_cascade_traits<ProdEval, Dst, Func>; + product_packet_assign(bool_constant<Traits::Enable>(), Func(), dst, lhs, rhs); } // This is a special evaluation path called from generic_product_impl<...,GemmProduct> in file GeneralMatrixMatrix.h @@ -829,6 +841,9 @@ using RhsPacketEtorType = std::conditional_t<bool(int(Flags) & PacketAccessBit), RhsEtorType, product_empty_packet_evaluator>; + static constexpr bool SingleTermCoeff = InnerSize == 1 && std::is_same<LhsPacketEtorType, LhsEtorType>::value && + std::is_same<RhsPacketEtorType, RhsEtorType>::value; + static constexpr int LhsOuterStrideBytes = int(LhsNestedCleaned::OuterStrideAtCompileTime) * int(sizeof(typename LhsNestedCleaned::Scalar)); static constexpr int RhsOuterStrideBytes = @@ -845,16 +860,17 @@ : RhsAlignment) : 0; - /* CanVectorizeInner deserves special explanation. It does not affect the product flags. It is not used outside - * of Product. If the Product itself is not a packet-access expression, there is still a chance that the inner - * loop of the product might be vectorized. This is the meaning of CanVectorizeInner. Since it doesn't affect - * the Flags, it is safe to make this value depend on ActualPacketAccessBit, that doesn't affect the ABI. - */ - static constexpr bool CanVectorizeInner = SameType && LhsRowMajor && (!RhsRowMajor) && - (int(LhsFlags) & int(RhsFlags) & ActualPacketAccessBit) && - (int(InnerSize) % packet_traits<Scalar>::size == 0); - EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE const CoeffReturnType coeff(Index row, Index col) const { + return coeff_impl(row, col, bool_constant<SingleTermCoeff>()); + } + + EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE CoeffReturnType coeff_impl(Index row, Index col, + std::true_type) const { + return fast_mult_op<LhsScalar, RhsScalar>()(m_lhsImpl.coeff(row, Index(0)), m_rhsImpl.coeff(Index(0), col)); + } + + EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE CoeffReturnType coeff_impl(Index row, Index col, + std::false_type) const { // fast_mult_op is cwiseProduct's scalar_product_op with a pmul-based scalar path, so the // reduction (and its precision) is unchanged but complex scalars avoid std::complex::operator* // (the slow libgcc __mul?c3). See fast_mult_op. @@ -868,7 +884,7 @@ EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE const CoeffReturnType coeff(Index index) const { const Index row = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime == 1) ? 0 : index; const Index col = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime == 1) ? index : 0; - return m_lhs.row(row).transpose().binaryExpr(m_rhs.col(col), fast_mult_op<LhsScalar, RhsScalar>()).sum(); + return coeff(row, col); } template <int LoadMode, typename PacketType> @@ -1373,14 +1389,16 @@ // off-triangle. Strip the scalar factor with blas_traits and re-fold it into // the kernel's alpha so the same scalar multiplies every output entry. using LhsBlasTraits = blas_traits<typename Lhs::MatrixType>; - using ActualLhsMatrix = decltype(LhsBlasTraits::extract(std::declval<const typename Lhs::MatrixType&>()) - .template conjugateIf<bool(LhsBlasTraits::NeedToConjugate)>()); + // A named constant, not bool(...): nvcc's front end re-emits that cast as a function type MSVC rejects. + static constexpr bool ConjLhs = LhsBlasTraits::NeedToConjugate; + using ActualLhsMatrix = + decltype(LhsBlasTraits::extract(std::declval<const typename Lhs::MatrixType&>()).template conjugateIf<ConjLhs>()); using ActualLhsMatrixType = remove_all_t<ActualLhsMatrix>; using Kernel = selfadjoint_diagonal_product_impl<Lhs::Mode, OnTheRight, ActualLhsMatrixType, typename Rhs::DiagonalVectorType>; static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ActualLhsMatrix actualLhsMatrix(const typename Lhs::MatrixType& matrix) { - return LhsBlasTraits::extract(matrix).template conjugateIf<bool(LhsBlasTraits::NeedToConjugate)>(); + return LhsBlasTraits::extract(matrix).template conjugateIf<ConjLhs>(); } template <typename Dest> @@ -1411,14 +1429,15 @@ // See note on the SelfAdjointShape, DiagonalShape specialization above for why // we extract the scalar factor with blas_traits. using RhsBlasTraits = blas_traits<typename Rhs::MatrixType>; - using ActualRhsMatrix = decltype(RhsBlasTraits::extract(std::declval<const typename Rhs::MatrixType&>()) - .template conjugateIf<bool(RhsBlasTraits::NeedToConjugate)>()); + static constexpr bool ConjRhs = RhsBlasTraits::NeedToConjugate; + using ActualRhsMatrix = + decltype(RhsBlasTraits::extract(std::declval<const typename Rhs::MatrixType&>()).template conjugateIf<ConjRhs>()); using ActualRhsMatrixType = remove_all_t<ActualRhsMatrix>; using Kernel = selfadjoint_diagonal_product_impl<Rhs::Mode, OnTheLeft, ActualRhsMatrixType, typename Lhs::DiagonalVectorType>; static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ActualRhsMatrix actualRhsMatrix(const typename Rhs::MatrixType& matrix) { - return RhsBlasTraits::extract(matrix).template conjugateIf<bool(RhsBlasTraits::NeedToConjugate)>(); + return RhsBlasTraits::extract(matrix).template conjugateIf<ConjRhs>(); } template <typename Dest> @@ -1812,8 +1831,10 @@ // Dense SelfAdjointView statically rejects the Upper|Lower mode (only one half is stored), so the // off-stored coefficient is always reconstructed by conjugating its mirror. -template <int Mode, int ProductOrder, typename MatrixType, typename DiagonalType, typename Derived> -struct selfadjoint_diagonal_product_lazy_evaluator_base : evaluator_base<Derived> { +template <int Mode, int ProductOrder, typename MatrixType, typename DiagonalType, typename ProductXpr, + bool Materialize = + NumTraits<typename MatrixType::Scalar>::IsComplex && blas_traits<MatrixType>::HasScalarFactor> +struct selfadjoint_diagonal_product_lazy_evaluator_base : evaluator_base<ProductXpr> { using Scalar = typename ScalarBinaryOpTraits<typename MatrixType::Scalar, typename DiagonalType::Scalar>::ReturnType; enum { @@ -1823,7 +1844,8 @@ Alignment = 0 }; - EIGEN_DEVICE_FUNC selfadjoint_diagonal_product_lazy_evaluator_base(const MatrixType& mat, const DiagonalType& diag) + EIGEN_DEVICE_FUNC selfadjoint_diagonal_product_lazy_evaluator_base(const ProductXpr&, const MatrixType& mat, + const DiagonalType& diag) : m_diagImpl(diag), m_matImpl(mat) {} EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index row, Index col) const { @@ -1839,34 +1861,44 @@ evaluator<MatrixType> m_matImpl; }; +// Extract complex factors before conjugating mirrored entries; the temporary also protects nested aliases. +// The parameter is not named XprType: MSVC lets the dependent base's XprType (an EvalToTemp) hide it. +template <int Mode, int ProductOrder, typename MatrixType, typename DiagonalType, typename ProductXpr> +struct selfadjoint_diagonal_product_lazy_evaluator_base<Mode, ProductOrder, MatrixType, DiagonalType, ProductXpr, true> + : evaluator<EvalToTemp<Product<typename ProductXpr::Lhs, typename ProductXpr::Rhs, DefaultProduct>>> { + using DefaultProductType = Product<typename ProductXpr::Lhs, typename ProductXpr::Rhs, DefaultProduct>; + using Base = evaluator<EvalToTemp<DefaultProductType>>; + EIGEN_DEVICE_FUNC selfadjoint_diagonal_product_lazy_evaluator_base(const ProductXpr& xpr, const MatrixType&, + const DiagonalType&) + : Base(DefaultProductType(xpr.lhs(), xpr.rhs())) {} +}; + // SelfAdjoint × Diagonal template <typename Lhs, typename Rhs, int ProductKind, int ProductTag> struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, SelfAdjointShape, DiagonalShape> - : selfadjoint_diagonal_product_lazy_evaluator_base< - Lhs::Mode, OnTheRight, typename Lhs::MatrixType, typename Rhs::DiagonalVectorType, - product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, SelfAdjointShape, DiagonalShape>> { + : selfadjoint_diagonal_product_lazy_evaluator_base<Lhs::Mode, OnTheRight, typename Lhs::MatrixType, + typename Rhs::DiagonalVectorType, + Product<Lhs, Rhs, ProductKind>> { using XprType = Product<Lhs, Rhs, ProductKind>; - using Base = selfadjoint_diagonal_product_lazy_evaluator_base< - Lhs::Mode, OnTheRight, typename Lhs::MatrixType, typename Rhs::DiagonalVectorType, - product_evaluator<XprType, ProductTag, SelfAdjointShape, DiagonalShape>>; + using Base = selfadjoint_diagonal_product_lazy_evaluator_base<Lhs::Mode, OnTheRight, typename Lhs::MatrixType, + typename Rhs::DiagonalVectorType, XprType>; EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr) - : Base(xpr.lhs().nestedExpression(), xpr.rhs().diagonal()) {} + : Base(xpr, xpr.lhs().nestedExpression(), xpr.rhs().diagonal()) {} }; // Diagonal × SelfAdjoint template <typename Lhs, typename Rhs, int ProductKind, int ProductTag> struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DiagonalShape, SelfAdjointShape> - : selfadjoint_diagonal_product_lazy_evaluator_base< - Rhs::Mode, OnTheLeft, typename Rhs::MatrixType, typename Lhs::DiagonalVectorType, - product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DiagonalShape, SelfAdjointShape>> { + : selfadjoint_diagonal_product_lazy_evaluator_base<Rhs::Mode, OnTheLeft, typename Rhs::MatrixType, + typename Lhs::DiagonalVectorType, + Product<Lhs, Rhs, ProductKind>> { using XprType = Product<Lhs, Rhs, ProductKind>; - using Base = selfadjoint_diagonal_product_lazy_evaluator_base< - Rhs::Mode, OnTheLeft, typename Rhs::MatrixType, typename Lhs::DiagonalVectorType, - product_evaluator<XprType, ProductTag, DiagonalShape, SelfAdjointShape>>; + using Base = selfadjoint_diagonal_product_lazy_evaluator_base<Rhs::Mode, OnTheLeft, typename Rhs::MatrixType, + typename Lhs::DiagonalVectorType, XprType>; EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr) - : Base(xpr.rhs().nestedExpression(), xpr.lhs().diagonal()) {} + : Base(xpr, xpr.rhs().nestedExpression(), xpr.lhs().diagonal()) {} }; /*************************************************************************** @@ -1876,7 +1908,8 @@ /** \internal * \class permutation_matrix_product * Internal helper class implementing the product between a permutation matrix and a matrix. - * This class is specialized for DenseShape below and for SparseShape in SparseCore/SparsePermutation.h. + * This class is specialized for DenseShape, TriangularShape and SelfAdjointShape below and for SparseShape in + * SparseCore/SparsePermutation.h. * The generic_product_impl cells for PermutationShape accept any operand shape, so the primary template * turns a missing specialization into a static assertion instead of an incomplete-type error. */ @@ -1944,6 +1977,30 @@ } }; +/** \internal + * Permutation products with a triangular or self-adjoint operand: the operand is evaluated into a plain dense + * matrix (its implicit unit diagonal, zeros and mirrored half included) and the dense kernel permutes that into + * dst. The result is dense; the permuted structure is not tracked. + */ +template <typename ExpressionType, int Side, bool Transposed> +struct permutation_densified_matrix_product { + using DenseType = typename ExpressionType::DenseMatrixType; + + template <typename Dest, typename PermutationType> + static EIGEN_DEVICE_FUNC void run(Dest& dst, const PermutationType& perm, const ExpressionType& xpr) { + const DenseType dense(xpr); + permutation_matrix_product<DenseType, Side, Transposed, DenseShape>::run(dst, perm, dense); + } +}; + +template <typename ExpressionType, int Side, bool Transposed> +struct permutation_matrix_product<ExpressionType, Side, Transposed, TriangularShape> + : permutation_densified_matrix_product<ExpressionType, Side, Transposed> {}; + +template <typename ExpressionType, int Side, bool Transposed> +struct permutation_matrix_product<ExpressionType, Side, Transposed, SelfAdjointShape> + : permutation_densified_matrix_product<ExpressionType, Side, Transposed> {}; + template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape> struct generic_product_impl<Lhs, Rhs, PermutationShape, MatrixShape, ProductTag> { template <typename Dest>
diff --git a/Eigen/src/Core/RealView.h b/Eigen/src/Core/RealView.h index f0005e1..bea630c 100644 --- a/Eigen/src/Core/RealView.h +++ b/Eigen/src/Core/RealView.h
@@ -43,7 +43,9 @@ using Scalar = typename NumTraits<ComplexScalar>::Real; static constexpr bool ArrayAccess = complex_array_access<ComplexScalar>::value; - static constexpr int ActualDirectAccessBit = ArrayAccess ? DirectAccessBit : 0; + // Real and imaginary parts interleave with a uniform stride only when the complex coefficients are adjacent. + static constexpr int ActualDirectAccessBit = + ArrayAccess && inner_stride_at_compile_time<Xpr>::value == 1 ? DirectAccessBit : 0; static constexpr int ActualLvalueBit = !std::is_const<Xpr>::value && ArrayAccess ? LvalueBit : 0; static constexpr int ActualPacketAccessBit = packet_traits<Scalar>::Vectorizable ? PacketAccessBit : 0; static constexpr int FlagMask = @@ -73,14 +75,16 @@ static constexpr int CoeffReadCost = BaseEvaluator::CoeffReadCost; static constexpr int Alignment = BaseEvaluator::Alignment; static constexpr bool IsRowMajor = ExpressionTraits::IsRowMajor; - static constexpr bool DirectAccess = (Flags & DirectAccessBit) != 0; + // DenseCoeffsBase::CoeffReturnType is a reference for LvalueBit as well as DirectAccessBit; returning a value + // there would dangle. + static constexpr bool CoeffByReference = (Flags & (LvalueBit | DirectAccessBit)) != 0; - using ComplexCoeffReturnType = std::conditional_t<DirectAccess, const ComplexScalar&, ComplexScalar>; - using CoeffReturnType = std::conditional_t<DirectAccess, const Scalar&, Scalar>; + using ComplexCoeffReturnType = std::conditional_t<CoeffByReference, const ComplexScalar&, ComplexScalar>; + using CoeffReturnType = std::conditional_t<CoeffByReference, const Scalar&, Scalar>; EIGEN_DEVICE_FUNC explicit evaluator(XprType realView) : BaseEvaluator(realView.m_xpr) {} - template <bool Enable = DirectAccess, std::enable_if_t<!Enable, bool> = true> + template <bool Enable = CoeffByReference, std::enable_if_t<!Enable, bool> = true> constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index row, Index col) const { Index r = IsRowMajor ? row : row / 2; Index c = IsRowMajor ? col / 2 : col; @@ -88,7 +92,7 @@ ComplexScalar ccoeff = BaseEvaluator::coeff(r, c); return p ? numext::imag(ccoeff) : numext::real(ccoeff); } - template <bool Enable = DirectAccess, std::enable_if_t<Enable, bool> = true> + template <bool Enable = CoeffByReference, std::enable_if_t<Enable, bool> = true> constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { Index r = IsRowMajor ? row : row / 2; Index c = IsRowMajor ? col / 2 : col; @@ -96,13 +100,13 @@ ComplexCoeffReturnType ccoeff = BaseEvaluator::coeff(r, c); return reinterpret_cast<const Scalar(&)[2]>(ccoeff)[p]; } - template <bool Enable = DirectAccess, std::enable_if_t<!Enable, bool> = true> + template <bool Enable = CoeffByReference, std::enable_if_t<!Enable, bool> = true> constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index index) const { ComplexScalar ccoeff = BaseEvaluator::coeff(index / 2); bool p = index & 1; return p ? numext::imag(ccoeff) : numext::real(ccoeff); } - template <bool Enable = DirectAccess, std::enable_if_t<Enable, bool> = true> + template <bool Enable = CoeffByReference, std::enable_if_t<Enable, bool> = true> constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const { ComplexCoeffReturnType ccoeff = BaseEvaluator::coeff(index / 2); Index p = index & 1; @@ -240,7 +244,10 @@ m_xpr.resize(Xpr::IsRowMajor ? rows : rows / 2, Xpr::IsRowMajor ? cols / 2 : cols); } EIGEN_DEVICE_FUNC void resize(Index size) { m_xpr.resize(size / 2); } - EIGEN_DEVICE_FUNC Scalar* data() { return reinterpret_cast<Scalar*>(m_xpr.data()); } + using ScalarWithConstIfNotLvalue = std::conditional_t<internal::is_lvalue<Xpr>::value, Scalar, const Scalar>; + EIGEN_DEVICE_FUNC ScalarWithConstIfNotLvalue* data() { + return reinterpret_cast<ScalarWithConstIfNotLvalue*>(m_xpr.data()); + } EIGEN_DEVICE_FUNC const Scalar* data() const { return reinterpret_cast<const Scalar*>(m_xpr.data()); } EIGEN_DEVICE_FUNC RealView(const RealView&) = default;
diff --git a/Eigen/src/Core/Redux.h b/Eigen/src/Core/Redux.h index 241ef1f..972d01a 100644 --- a/Eigen/src/Core/Redux.h +++ b/Eigen/src/Core/Redux.h
@@ -46,6 +46,15 @@ min_size_prefer_fixed(min_size_prefer_fixed(Left::Size, Right::Size), size_at_compile_time(Rows, Cols)); }; +template <typename Op, typename Arg> +struct redux_max_size<CwiseUnaryOp<Op, Arg>> : redux_max_size<remove_all_t<Arg>> {}; + +template <typename Arg> +struct redux_max_size<ArrayWrapper<Arg>> : redux_max_size<remove_all_t<Arg>> {}; + +template <typename Arg> +struct redux_max_size<MatrixWrapper<Arg>> : redux_max_size<remove_all_t<Arg>> {}; + template <typename Op, typename Arg1, typename Arg2, typename Arg3> struct redux_max_size<CwiseTernaryOp<Op, Arg1, Arg2, Arg3>> { using First = redux_max_size<remove_all_t<Arg1>>; @@ -581,15 +590,17 @@ packet_res2 = func.packetOp(packet_res2, packetAt(eval, 2, i)); packet_res3 = func.packetOp(packet_res3, packetAt(eval, 3, i)); } - Index j = 4; - for (; j + 4 <= outerSize; j += 4) + // Both loops take their bounds from quadOuterSize rather than sharing j: for a fixed outer size, GCC 10 + // otherwise emits -Waggressive-loop-optimizations for the tail loop, which never runs. + const Index quadOuterSize = numext::round_down(outerSize, 4); + for (Index j = 4; j < quadOuterSize; j += 4) for (Index i = 0; i < packetedInnerSize; i += PacketSize) { packet_res0 = func.packetOp(packet_res0, packetAt(eval, j, i)); packet_res1 = func.packetOp(packet_res1, packetAt(eval, j + 1, i)); packet_res2 = func.packetOp(packet_res2, packetAt(eval, j + 2, i)); packet_res3 = func.packetOp(packet_res3, packetAt(eval, j + 3, i)); } - for (; j < outerSize; ++j) + for (Index j = quadOuterSize; j < outerSize; ++j) for (Index i = 0; i < packetedInnerSize; i += PacketSize) packet_res0 = func.packetOp(packet_res0, packetAt(eval, j, i));
diff --git a/Eigen/src/Core/Replicate.h b/Eigen/src/Core/Replicate.h index 322b067..932acb9 100644 --- a/Eigen/src/Core/Replicate.h +++ b/Eigen/src/Core/Replicate.h
@@ -103,7 +103,8 @@ * Example: \include MatrixBase_replicate.cpp * Output: \verbinclude MatrixBase_replicate.out * - * \sa VectorwiseOp::replicate(), DenseBase::replicate(Index,Index), class Replicate + * \sa VectorwiseOp::replicate(), DenseBase::replicate(Index,Index) const, + * DenseBase::replicate(NRowsType,NColsType) const, class Replicate */ template <typename Derived> template <int RowFactor, int ColFactor> @@ -117,7 +118,7 @@ * Example: \include DirectionWise_replicate_int.cpp * Output: \verbinclude DirectionWise_replicate_int.out * - * \sa VectorwiseOp::replicate(), DenseBase::replicate(), class Replicate + * \sa VectorwiseOp::replicate(), VectorwiseOp::replicate(NFactorType) const, DenseBase::replicate(), class Replicate */ template <typename ExpressionType, int Direction> EIGEN_DEVICE_FUNC const typename VectorwiseOp<ExpressionType, Direction>::ReplicateReturnType
diff --git a/Eigen/src/Core/SafeScaling.h b/Eigen/src/Core/SafeScaling.h index 8849776..ddc3b83 100644 --- a/Eigen/src/Core/SafeScaling.h +++ b/Eigen/src/Core/SafeScaling.h
@@ -42,35 +42,43 @@ Scalar invScale = Scalar(1); }; +// value * 2^exponent through the integer significand of the value with bit pattern valueBits, so FTZ/DAZ hardware and +// ARMv7 NEON can flush neither a subnormal input nor a subnormal result. Exact wherever the result is representable; +// a subnormal result rounds to nearest, ties to even, as IEEE 754 multiplication does; below denorm_min / 2 the result +// is a signed zero and above the largest finite value a signed infinity. Zeros, infinities and NaNs pass through. Keep +// the integer-only ABI so compilers cannot replace the reconstruction with FTZ/DAZ-sensitive floating-point arithmetic. template <typename Scalar> -EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE typename binary_floating_point_traits<Scalar>::Bits -scale_binary_bits_by_power_of_two(const typename binary_floating_point_traits<Scalar>::Bits valueBits, - const typename binary_floating_point_traits<Scalar>::Bits factorExponentBits) { +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE typename binary_floating_point_traits<Scalar>::Bits scale_binary_bits_by_exponent( + const typename binary_floating_point_traits<Scalar>::Bits valueBits, int exponent) { using Binary = binary_floating_point_traits<Scalar>; using Bits = typename Binary::Bits; + constexpr int kInfinityExponent = int(Binary::kExponentMask >> Binary::kFractionBits); const Bits valueExponentBits = valueBits & Binary::kExponentMask; const Bits fraction = valueBits & Binary::kFractionMask; if (valueExponentBits == Binary::kExponentMask || (valueExponentBits == 0 && fraction == 0)) return valueBits; + const Bits sign = valueBits & Binary::kSignBit; Bits significand = fraction; - int exponent = int(valueExponentBits >> Binary::kFractionBits); - if (exponent == 0) { - exponent = 1; + int resultExponent = int(valueExponentBits >> Binary::kFractionBits); + if (resultExponent == 0) { + resultExponent = 1; while (significand < Binary::kExponentUnit) { significand <<= 1; - --exponent; + --resultExponent; } } - exponent += int(factorExponentBits >> Binary::kFractionBits) - Binary::kExponentBias; - eigen_internal_assert(exponent < int(Binary::kExponentMask >> Binary::kFractionBits)); - const Bits sign = valueBits & Binary::kSignBit; - if (exponent > 0) return sign | (Bits(exponent) << Binary::kFractionBits) | (significand & Binary::kFractionMask); + // Every finite value saturates beyond this magnitude of exponent, so the clamp changes no result. + constexpr int kSaturatingExponent = kInfinityExponent + Binary::kFractionBits + 2; + resultExponent += numext::mini(numext::maxi(exponent, -kSaturatingExponent), kSaturatingExponent); + if (resultExponent >= kInfinityExponent) return sign | Binary::kExponentMask; + if (resultExponent > 0) + return sign | (Bits(resultExponent) << Binary::kFractionBits) | (significand & Binary::kFractionMask); - // Subnormal result m * 2^(exponent - 1) denorm_min, m = significand | kExponentUnit < 2 * kExponentUnit: round to - // nearest, ties to even, as IEEE 754 multiplication does. A carry into kExponentUnit encodes the smallest normal; for - // shift > kFractionBits + 1 the result is below denorm_min / 2 and rounds to a signed zero. - const int shift = 1 - exponent; + // Subnormal result m * 2^(resultExponent - 1) denorm_min, m = significand | kExponentUnit < 2 * kExponentUnit: round + // to nearest, ties to even, as IEEE 754 multiplication does. A carry into kExponentUnit encodes the smallest normal; + // for shift > kFractionBits + 1 the result is below denorm_min / 2 and rounds to a signed zero. + const int shift = 1 - resultExponent; if (shift > Binary::kFractionBits + 1) return sign; const Bits mantissa = significand | Binary::kExponentUnit; const Bits halfway = Bits(1) << (shift - 1); @@ -80,9 +88,80 @@ return sign | rounded; } +// The same scaling by a positive normal power of two given as the bit pattern of its exponent field. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename binary_floating_point_traits<Scalar>::Bits +scale_binary_bits_by_power_of_two(const typename binary_floating_point_traits<Scalar>::Bits valueBits, + const typename binary_floating_point_traits<Scalar>::Bits factorExponentBits) { + using Binary = binary_floating_point_traits<Scalar>; + return scale_binary_bits_by_exponent<Scalar>( + valueBits, int(factorExponentBits >> Binary::kFractionBits) - Binary::kExponentBias); +} + +// value * 2^exponent for the binary floating-point scalars, with the rounding and saturation of +// scale_binary_bits_by_exponent(): the ldexp that FTZ/DAZ hardware cannot flush, for any int exponent. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar scale_binary_by_exponent(const Scalar& value, int exponent) { + using Binary = binary_floating_point_traits<Scalar>; + return numext::bit_cast<Scalar>(scale_binary_bits_by_exponent<Scalar>(Binary::bits(value), exponent)); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<Scalar> scale_binary_by_exponent(const std::complex<Scalar>& value, + int exponent) { + return std::complex<Scalar>(scale_binary_by_exponent(value.real(), exponent), + scale_binary_by_exponent(value.imag(), exponent)); +} + +// frexp's exponent e, 2^(e - 1) <= |value| < 2^e, read from the representation of a finite value so that FTZ/DAZ +// hardware cannot flush a subnormal; 0 for a zero. Infinities and NaNs are not classified: they come back one above +// the largest finite exponent. The classification takes the magnitude bits out of line, like +// scale_binary_bits_by_exponent(): inlined, a compiler folds an integer test on the bit pattern of a float back into +// a floating-point comparison (`magnitude == 0` into `value == 0`), which FTZ/DAZ hardware flushes exactly like the +// arithmetic the bits were meant to bypass. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE int binary_frexp_exponent_of_magnitude( + const typename binary_floating_point_traits<Scalar>::Bits magnitude) { + using Binary = binary_floating_point_traits<Scalar>; + using Bits = typename Binary::Bits; + if (magnitude == 0) return 0; + const Bits exponentBits = magnitude & Binary::kExponentMask; + if (exponentBits != 0) return int(exponentBits >> Binary::kFractionBits) - Binary::kExponentBias + 1; + // A subnormal is its fraction times 2^(min_exponent - digits). + return log_2_impl<Bits>::run_floor(magnitude) + 1 + std::numeric_limits<Scalar>::min_exponent - + std::numeric_limits<Scalar>::digits; +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int binary_frexp_exponent(const Scalar& value) { + return binary_frexp_exponent_of_magnitude<Scalar>(binary_floating_point_traits<Scalar>::magnitude(value)); +} + +// The larger of two magnitude bit patterns, out of line for the same reason. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE typename binary_floating_point_traits<Scalar>::Bits larger_magnitude_bits( + const typename binary_floating_point_traits<Scalar>::Bits a, + const typename binary_floating_point_traits<Scalar>::Bits b) { + return a < b ? b : a; +} + +// |value| < the smallest normal from the magnitude bits, out of line for the same reason. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE bool is_below_normal_magnitude_bits( + const typename binary_floating_point_traits<Scalar>::Bits magnitude) { + return magnitude < binary_floating_point_traits<Scalar>::kExponentUnit; +} + +// frexp through the representation: the significand in [0.5, 1), with the sign of value, and its exponent. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar binary_frexp(const Scalar& value, int& exponent) { + exponent = binary_frexp_exponent(value); + return scale_binary_by_exponent(value, -exponent); +} + // Multiplication by a positive normal power of two through integer significands, so FTZ/DAZ cannot flush a subnormal -// input or result; a subnormal result rounds as IEEE 754 multiplication does, and no result may overflow. Keep the -// integer-only ABI so compilers cannot replace reconstruction with FTZ/DAZ-sensitive floating-point arithmetic. +// input or result; a subnormal result rounds as IEEE 754 multiplication does, and a result above the largest finite +// value saturates to infinity. template <typename Scalar> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar scale_binary_by_power_of_two(const Scalar& value, const Scalar& factor) { using Binary = binary_floating_point_traits<Scalar>; @@ -100,6 +179,26 @@ scale_binary_by_power_of_two(value.imag(), factor)); } +// Coefficient-wise value * 2^exponent through scale_binary_by_exponent(). Scalar only: the integer path is the point. +template <typename Scalar> +struct scale_by_exponent_op { + EIGEN_DEVICE_FUNC explicit scale_by_exponent_op(int exponent) : m_exponent(exponent) {} + + template <typename CoeffScalar> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffScalar operator()(const CoeffScalar& value) const { + return scale_binary_by_exponent(value, m_exponent); + } + + int m_exponent; +}; + +template <typename Scalar> +struct functor_traits<scale_by_exponent_op<Scalar>> { + static constexpr int Cost = 10 * NumTraits<Scalar>::MulCost; + static constexpr bool PacketAccess = false; + static constexpr bool IsRepeatable = true; +}; + template <typename FactorScalar> struct scale_by_power_of_two_op { EIGEN_DEVICE_FUNC explicit scale_by_power_of_two_op(const FactorScalar& factor) : m_factor(factor) {} @@ -128,6 +227,154 @@ (std::is_same<FactorScalar, double>::value && (std::is_same<CoeffScalar, double>::value || std::is_same<CoeffScalar, std::complex<double>>::value))> {}; +// Subnormal-magnitude tests read the representation for float and double, where a comparison under DAZ +// reads a subnormal as zero as well; the other scalars compare arithmetically. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_subnormal_magnitude_impl(const Scalar& value, true_type) { + // The smallest normal, 2^(min_exponent - 1), has frexp exponent min_exponent. + const int exponent = binary_frexp_exponent(value); + return exponent != 0 && exponent < std::numeric_limits<Scalar>::min_exponent; +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_subnormal_magnitude_impl(const Scalar& value, false_type) { + const Scalar magnitude = numext::abs(value); + return magnitude > Scalar(0) && magnitude < (std::numeric_limits<Scalar>::min)(); +} +// 0 < |value| < the smallest normal. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_subnormal_magnitude(const Scalar& value) { + return is_subnormal_magnitude_impl(value, bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>()); +} + +// |value| < the smallest normal, zero included: a maximum that a flushing comparison may have produced. A scalar +// reduction under FTZ/DAZ does not always return a zero: where max(a, b) is a compare and a select (Arm FZ with +// scalar VFP or fcsel, ARMv7's double, which has no packet), every comparison is false and the running maximum keeps +// the bits of the first operand. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_zero_or_subnormal_magnitude_impl(const Scalar& value, true_type) { + return is_below_normal_magnitude_bits<Scalar>(binary_floating_point_traits<Scalar>::magnitude(value)); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_zero_or_subnormal_magnitude_impl(const Scalar& value, false_type) { + return numext::abs(value) < (std::numeric_limits<Scalar>::min)(); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_zero_or_subnormal_magnitude(const Scalar& value) { + return is_zero_or_subnormal_magnitude_impl(value, + bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>()); +} + +// The larger of two non-negative values, such as two recovered maxima, read from the representation for float and +// double: an arithmetic maximum (a compiler may lower it to fmaxnm) flushes subnormal operands to zero under FTZ/DAZ. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar max_preserving_subnormals_impl(const Scalar& a, const Scalar& b, + true_type) { + using Binary = binary_floating_point_traits<Scalar>; + return numext::bit_cast<Scalar>(larger_magnitude_bits<Scalar>(Binary::magnitude(a), Binary::magnitude(b))); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar max_preserving_subnormals_impl(const Scalar& a, const Scalar& b, + false_type) { + return numext::maxi(a, b); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar max_preserving_subnormals(const Scalar& a, const Scalar& b) { + return max_preserving_subnormals_impl(a, b, bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>()); +} + +// |value| and value < 0 of a finite real scalar, from the representation for float and double: a comparison reads a +// negative subnormal as zero under DAZ, and an abs() that widens (MSVC 19.29 takes abs(float) through double) flushes +// a subnormal result under FTZ when it narrows back. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar abs_preserving_subnormals_impl(const Scalar& value, true_type) { + return numext::bit_cast<Scalar>(binary_floating_point_traits<Scalar>::magnitude(value)); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar abs_preserving_subnormals_impl(const Scalar& value, false_type) { + return numext::abs(value); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar abs_preserving_subnormals(const Scalar& value) { + return abs_preserving_subnormals_impl(value, + bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>()); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_negative_preserving_subnormals_impl(const Scalar& value, true_type) { + using Binary = binary_floating_point_traits<Scalar>; + return (Binary::bits(value) & Binary::kSignBit) != 0 && !is_zero_magnitude_bits<Scalar>(Binary::magnitude(value)); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_negative_preserving_subnormals_impl(const Scalar& value, false_type) { + return value < Scalar(0); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_negative_preserving_subnormals(const Scalar& value) { + return is_negative_preserving_subnormals_impl( + value, bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>()); +} + +// The position of the largest magnitude of a real vector with at least one coefficient, read from the representation +// and out of line like the other bit classifications. +template <typename Scalar, typename Derived> +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE Index index_of_largest_magnitude(const DenseBase<Derived>& src) { + using Binary = binary_floating_point_traits<Scalar>; + using Bits = typename Binary::Bits; + const evaluator<Derived> coeffs(src.derived()); + Index position = 0; + Bits largest = Binary::magnitude(coeffs.coeff(0)); + for (Index i = 1; i < src.size(); ++i) { + const Bits magnitude = Binary::magnitude(coeffs.coeff(i)); + if (magnitude > largest) { + largest = magnitude; + position = i; + } + } + return position; +} + +// ldexp and frexp's exponent for a real scalar: float and double through the representation, which FTZ/DAZ cannot +// reach, the other scalars through the C library. frexp's exponent is e with 2^(e - 1) <= |value| < 2^e, 0 for a +// zero. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar ldexp_preserving_subnormals_impl(const Scalar& value, int exponent, + true_type) { + return scale_binary_by_exponent(value, exponent); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar ldexp_preserving_subnormals_impl(const Scalar& value, int exponent, + false_type) { + return numext::ldexp(value, exponent); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar ldexp_preserving_subnormals(const Scalar& value, int exponent) { + return ldexp_preserving_subnormals_impl(value, exponent, + bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>()); +} + +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int frexp_exponent_preserving_subnormals_impl(const Scalar& value, true_type) { + return binary_frexp_exponent(value); +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int frexp_exponent_preserving_subnormals_impl(const Scalar& value, false_type) { + int exponent = 0; + EIGEN_USING_STD(frexp); + frexp(value, &exponent); + return exponent; +} +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int frexp_exponent_preserving_subnormals(const Scalar& value) { + return frexp_exponent_preserving_subnormals_impl( + value, bool_constant<use_subnormal_preserving_scaling<Scalar, Scalar>::value>()); +} +// The significand in [0.5, 1), with the sign of value, and its exponent. +template <typename Scalar> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar frexp_preserving_subnormals(const Scalar& value, int& exponent) { + exponent = frexp_exponent_preserving_subnormals(value); + return ldexp_preserving_subnormals(value, -exponent); +} + template <typename Scalar, bool IsPowerOfTwo_> struct safe_scaling_operations { private: @@ -136,8 +383,7 @@ EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE bool needs_subnormal_recovery(const Scalar& value) { using Binary = binary_floating_point_traits<Scalar>; using Bits = typename Binary::Bits; - constexpr int kDirectScaleExponent = std::numeric_limits<Scalar>::min_exponent + Binary::kFractionBits - 1; - constexpr Bits kThreshold = Bits(kDirectScaleExponent + Binary::kExponentBias) << Binary::kFractionBits; + constexpr Bits kThreshold = Bits(subnormal_recovery_exponent() + Binary::kExponentBias) << Binary::kFractionBits; return Binary::magnitude(value) - Bits(1) < kThreshold - Bits(1); } @@ -224,7 +470,7 @@ true_type) { using Binary = binary_floating_point_traits<Scalar>; using Bits = typename Binary::Bits; - if (Binary::magnitude(maxCoeff) != 0) return maxCoeff; + if (!is_zero_or_subnormal_magnitude(maxCoeff)) return maxCoeff; // A product expression has no coefficient access; its evaluator materializes it. const evaluator<Src> coeffs(src); Bits maxBits = 0; @@ -238,6 +484,18 @@ return numext::bit_cast<Scalar>(maxBits); } + template <typename Derived> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index recover_flushed_max_coeff_index_impl(const DenseBase<Derived>&, + Index position, false_type) { + return position; + } + + template <typename Derived> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index recover_flushed_max_coeff_index_impl(const DenseBase<Derived>& src, + Index, true_type) { + return index_of_largest_magnitude<Scalar>(src); + } + public: EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Factors compute_ceiling_factors_with_normal_reciprocal(const Scalar& value) { @@ -304,9 +562,17 @@ unscale_to_impl(dest, src, maxCoeff, factors, bool_constant<kPreserveSubnormalOutputs>()); } + // Below 2^subnormal_recovery_exponent(), a coefficient within a factor 2^-kFractionBits of the largest one can be + // subnormal, so a scaling that must survive FTZ/DAZ goes through integer significands (scale_by_exponent_op, + // scale_by_power_of_two_op) instead of packet multiplication. + EIGEN_DEVICE_FUNC static constexpr int subnormal_recovery_exponent() { + return std::numeric_limits<Scalar>::min_exponent + std::numeric_limits<Scalar>::digits - 2; + } + // A SIMD unit that flushes subnormal inputs (ARMv7 NEON, Arm FZ, DAZ) reduces an all-subnormal matrix to a zero - // maximum, so rescan a zero maxCoeff from the representation. Any subnormal maximum, including the largest real or - // imaginary magnitude of a complex matrix, selects the same factors as the true one. + // maximum, and a flushed scalar reduction to whichever subnormal it held first, so rescan a zero or subnormal + // maxCoeff from the representation. Any subnormal maximum, including the largest real or imaginary magnitude of a + // complex matrix, selects the same factors as the true one. template <typename Src> EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar recover_flushed_max_coeff(const Src& src, const Scalar& maxCoeff) { @@ -315,6 +581,17 @@ return recover_flushed_max_coeff_impl(src, maxCoeff, bool_constant<kPreserveSubnormalInputs>()); } + // The position of the largest magnitude in a real vector whose maxCoeff(&position) read zero or subnormal, + // rescanned from the representation like recover_flushed_max_coeff(); the position maxCoeff() reported where there + // is no representation path. + template <typename Derived> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index recover_flushed_max_coeff_index(const DenseBase<Derived>& src, + Index position) { + constexpr bool kPreserveSubnormalInputs = + IsPowerOfTwo_ && use_subnormal_preserving_scaling<Scalar, typename Derived::Scalar>::value; + return recover_flushed_max_coeff_index_impl(src, position, bool_constant<kPreserveSubnormalInputs>()); + } + template <typename MatrixType> EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Factors scale_in_place(MatrixType& matrix, const Scalar& maxCoeff) { const Factors factors = select_factors(maxCoeff);
diff --git a/Eigen/src/Core/Swap.h b/Eigen/src/Core/Swap.h index 6190eea..3372fb2 100644 --- a/Eigen/src/Core/Swap.h +++ b/Eigen/src/Core/Swap.h
@@ -42,7 +42,7 @@ : Base(dst, src, func, dstExpr) {} template <int StoreMode, int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE void assignPacket(Index row, Index col) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index row, Index col) { PacketType tmp = m_src.template packet<LoadMode, PacketType>(row, col); const_cast<SrcEvaluatorTypeT &>(m_src).template writePacket<LoadMode>( row, col, m_dst.template packet<StoreMode, PacketType>(row, col)); @@ -50,7 +50,7 @@ } template <int StoreMode, int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE void assignPacket(Index index) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index index) { PacketType tmp = m_src.template packet<LoadMode, PacketType>(index); const_cast<SrcEvaluatorTypeT &>(m_src).template writePacket<LoadMode>( index, m_dst.template packet<StoreMode, PacketType>(index)); @@ -58,14 +58,14 @@ } template <int StoreMode, int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner) { Index row = Base::rowIndexByOuterInner(outer, inner); Index col = Base::colIndexByOuterInner(outer, inner); assignPacket<StoreMode, LoadMode, PacketType>(row, col); } template <int StoreMode, int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE void assignPacketSegment(Index row, Index col, Index begin, Index count) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketSegment(Index row, Index col, Index begin, Index count) { PacketType tmp = m_src.template packetSegment<LoadMode, PacketType>(row, col, begin, count); const_cast<SrcEvaluatorTypeT &>(m_src).template writePacketSegment<LoadMode>( row, col, m_dst.template packetSegment<StoreMode, PacketType>(row, col, begin, count), begin, count); @@ -73,7 +73,7 @@ } template <int StoreMode, int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE void assignPacketSegment(Index index, Index begin, Index count) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketSegment(Index index, Index begin, Index count) { PacketType tmp = m_src.template packetSegment<LoadMode, PacketType>(index, begin, count); const_cast<SrcEvaluatorTypeT &>(m_src).template writePacketSegment<LoadMode>( index, m_dst.template packetSegment<StoreMode, PacketType>(index, begin, count), begin, count); @@ -81,7 +81,8 @@ } template <int StoreMode, int LoadMode, typename PacketType> - EIGEN_STRONG_INLINE void assignPacketSegmentByOuterInner(Index outer, Index inner, Index begin, Index count) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketSegmentByOuterInner(Index outer, Index inner, Index begin, + Index count) { Index row = Base::rowIndexByOuterInner(outer, inner); Index col = Base::colIndexByOuterInner(outer, inner); assignPacketSegment<StoreMode, LoadMode, PacketType>(row, col, begin, count);
diff --git a/Eigen/src/Core/Transpositions.h b/Eigen/src/Core/Transpositions.h index cf8a384..9d7db6b 100644 --- a/Eigen/src/Core/Transpositions.h +++ b/Eigen/src/Core/Transpositions.h
@@ -94,6 +94,10 @@ /** \returns the transpose transformation */ inline Transpose<TranspositionsBase> transpose() const { return Transpose<TranspositionsBase>(derived()); } + + /** \returns the adjoint transformation. Its matrix is a real permutation, so this equals transpose() and inverse(). + */ + Transpose<TranspositionsBase> adjoint() const { return Transpose<TranspositionsBase>(derived()); } }; namespace internal {
diff --git a/Eigen/src/Core/TriangularMatrix.h b/Eigen/src/Core/TriangularMatrix.h index 01bb45f..a7ad5f2 100644 --- a/Eigen/src/Core/TriangularMatrix.h +++ b/Eigen/src/Core/TriangularMatrix.h
@@ -239,6 +239,44 @@ other.derived().nestedExpression()); } + // Sums with a diagonal matrix keep the view's structure: the result is the same view of the lazy dense sum. + // A unit or zero diagonal is implicit in the view and would swallow the diagonal term, so those modes have + // no such operator; a self-adjoint view needs a real diagonal to stay self-adjoint. + + // The Mode_ = Mode parameter makes the condition depend on the operator's own template parameters, so an + // excluded mode removes the overload instead of failing the class instantiation. + + /** \returns the view of mode \c Mode of the sum of the nested expression and the diagonal matrix \a other */ + template <typename OtherDerived, unsigned int Mode_ = Mode, + std::enable_if_t<(int(Mode_) & (int(UnitDiag) | int(ZeroDiag))) == 0, int> = 0> + EIGEN_DEVICE_FUNC inline auto operator+(const DiagonalBase<OtherDerived>& other) const { + return internal::make_triangular_base_cwise_view<Mode>(derived().nestedExpression() + other.derived()); + } + + /** \returns the view of mode \c Mode of the difference of the nested expression and the diagonal matrix \a other + */ + template <typename OtherDerived, unsigned int Mode_ = Mode, + std::enable_if_t<(int(Mode_) & (int(UnitDiag) | int(ZeroDiag))) == 0, int> = 0> + EIGEN_DEVICE_FUNC inline auto operator-(const DiagonalBase<OtherDerived>& other) const { + return internal::make_triangular_base_cwise_view<Mode>(derived().nestedExpression() - other.derived()); + } + + /** \returns the view of mode \c Mode of the sum of the diagonal matrix \a lhs and the nested expression of \a rhs + */ + template <typename OtherDerived, unsigned int Mode_ = Mode, + std::enable_if_t<(int(Mode_) & (int(UnitDiag) | int(ZeroDiag))) == 0, int> = 0> + friend EIGEN_DEVICE_FUNC inline auto operator+(const DiagonalBase<OtherDerived>& lhs, const Derived& rhs) { + return internal::make_triangular_base_cwise_view<Mode>(lhs.derived() + rhs.nestedExpression()); + } + + /** \returns the view of mode \c Mode of the difference of the diagonal matrix \a lhs and the nested expression of + * \a rhs */ + template <typename OtherDerived, unsigned int Mode_ = Mode, + std::enable_if_t<(int(Mode_) & (int(UnitDiag) | int(ZeroDiag))) == 0, int> = 0> + friend EIGEN_DEVICE_FUNC inline auto operator-(const DiagonalBase<OtherDerived>& lhs, const Derived& rhs) { + return internal::make_triangular_base_cwise_view<Mode>(lhs.derived() - rhs.nestedExpression()); + } + template <typename OtherDerived> friend EIGEN_DEVICE_FUNC const Product<OtherDerived, Derived> operator*(const MatrixBase<OtherDerived>& lhs, const Derived& rhs) { @@ -251,6 +289,37 @@ return Product<OtherDerived, Derived>(lhs.derived(), rhs); } + // Products with permutations (and their inverses) evaluate to a dense matrix: the permuted view has no + // triangular or self-adjoint structure Eigen could represent. They live here rather than next to the + // MatrixBase permutation operators so that every operator* a TriangularBase takes part in is in one place. + + /** \returns the dense matrix expression of \c *this with the permutation \a rhs applied to its columns. */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC const Product<Derived, OtherDerived> operator*(const PermutationBase<OtherDerived>& rhs) const { + return Product<Derived, OtherDerived>(derived(), rhs.derived()); + } + + /** \returns the dense matrix expression of \c *this with the inverse permutation \a rhs applied to its columns. */ + template <typename OtherDerived> + EIGEN_DEVICE_FUNC const Product<Derived, Inverse<OtherDerived>> operator*( + const InverseImpl<OtherDerived, PermutationStorage>& rhs) const { + return Product<Derived, Inverse<OtherDerived>>(derived(), rhs.derived()); + } + + /** \returns the dense matrix expression of \a rhs with the permutation \a lhs applied to its rows. */ + template <typename OtherDerived> + friend EIGEN_DEVICE_FUNC const Product<OtherDerived, Derived> operator*(const PermutationBase<OtherDerived>& lhs, + const Derived& rhs) { + return Product<OtherDerived, Derived>(lhs.derived(), rhs); + } + + /** \returns the dense matrix expression of \a rhs with the inverse permutation \a lhs applied to its rows. */ + template <typename OtherDerived> + friend EIGEN_DEVICE_FUNC const Product<Inverse<OtherDerived>, Derived> operator*( + const InverseImpl<OtherDerived, PermutationStorage>& lhs, const Derived& rhs) { + return Product<Inverse<OtherDerived>, Derived>(lhs.derived(), rhs); + } + /** \sa MatrixBase::conjugate() const */ EIGEN_DEVICE_FUNC inline const ConjugateReturnType conjugate() const { return ConjugateReturnType(derived().nestedExpression().conjugate());
diff --git a/Eigen/src/Core/VectorwiseOp.h b/Eigen/src/Core/VectorwiseOp.h index f68f824..4c95def 100644 --- a/Eigen/src/Core/VectorwiseOp.h +++ b/Eigen/src/Core/VectorwiseOp.h
@@ -576,7 +576,8 @@ * Example: \include DirectionWise_replicate.cpp * Output: \verbinclude DirectionWise_replicate.out * - * \sa VectorwiseOp::replicate(Index), DenseBase::replicate(), class Replicate + * \sa VectorwiseOp::replicate(Index), VectorwiseOp::replicate(NFactorType) const, DenseBase::replicate(), + * class Replicate */ // NOTE implemented here because of sunstudio's compilation errors // isVertical*Factor+isHorizontal instead of (isVertical?Factor:1) to handle CUDA bug with ternary operator @@ -588,6 +589,23 @@ _expression(), isVertical ? factor : 1, isHorizontal ? factor : 1); } + /** + * \return an expression of the replication of each column (or row) of \c *this + * + * \param nFactor the number of repetitions, specified at run time or at compile time using Eigen::fix<N>. + * Eigen::fix<N>(n) uses \c n as a runtime fallback when \c N equals Eigen::Dynamic, and \c n must equal \c N + * otherwise. + * + * \sa VectorwiseOp::replicate(Index), DenseBase::replicate(), class Replicate, fix, fix<N>(int) + */ + template <typename NFactorType> + EIGEN_DEVICE_FUNC const + Replicate<ExpressionType, isVertical * internal::get_fixed_value<NFactorType>::value + isHorizontal, + isHorizontal * internal::get_fixed_value<NFactorType>::value + isVertical> + replicate(NFactorType nFactor) const { + return replicate<internal::get_fixed_value<NFactorType>::value>(internal::get_runtime_value(nFactor)); + } + /////////// Arithmetic operators /////////// // The broadcast (compound-)assignments below bind the rhs through
diff --git a/Eigen/src/Core/Visitor.h b/Eigen/src/Core/Visitor.h index 0c23d98..10893e5 100644 --- a/Eigen/src/Core/Visitor.h +++ b/Eigen/src/Core/Visitor.h
@@ -18,6 +18,13 @@ namespace internal { +// Opt in only when coefficient updates are valid without init/initpacket. +template <typename Visitor, typename = void> +struct visitor_already_initialized : std::false_type {}; +template <typename Visitor> +struct visitor_already_initialized<Visitor, void_t<decltype(functor_traits<Visitor>::AlreadyInitialized)>> + : bool_constant<functor_traits<Visitor>::AlreadyInitialized> {}; + template <typename Visitor, typename Derived, int UnrollCount, bool Vectorize = (Derived::PacketAccess && functor_traits<Visitor>::PacketAccess), bool LinearAccess = false, bool ShortCircuitEvaluation = false> @@ -197,12 +204,13 @@ static constexpr int PacketSize = packet_traits<Scalar>::size; using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>; static constexpr bool RowMajor = Derived::IsRowMajor; + static constexpr bool AlreadyInitialized = visitor_already_initialized<Visitor>::value; static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { const Index innerSize = RowMajor ? mat.cols() : mat.rows(); const Index outerSize = RowMajor ? mat.rows() : mat.cols(); if (innerSize == 0 || outerSize == 0) return; - { + EIGEN_IF_CONSTEXPR (!AlreadyInitialized) { Index i = 0; if (innerSize < PacketSize) { visitor.init(mat.coeff(0, 0), 0, 0); @@ -227,9 +235,10 @@ if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return; } } - for (Index j = 1; j < outerSize; j++) { + const Index packetEnd = innerSize - innerSize % PacketSize; + for (Index j = AlreadyInitialized ? 0 : 1; j < outerSize; j++) { Index i = 0; - for (; i + PacketSize - 1 < innerSize; i += PacketSize) { + for (; AlreadyInitialized ? i < packetEnd : i + PacketSize - 1 < innerSize; i += PacketSize) { Index r = RowMajor ? j : i; Index c = RowMajor ? i : j; Packet p = mat.template packet<Packet>(r, c); @@ -270,21 +279,24 @@ using Packet = typename packet_traits<Scalar>::type; static constexpr int PacketSize = packet_traits<Scalar>::size; using short_circuit = short_circuit_eval_impl<Visitor, ShortCircuitEvaluation>; + static constexpr bool AlreadyInitialized = visitor_already_initialized<Visitor>::value; static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& mat, Visitor& visitor) { const Index size = mat.size(); if (size == 0) return; const Index packetEnd = size - size % PacketSize; Index k = 0; - if (size < PacketSize) { - visitor.init(mat.coeff(0), 0); - k = 1; - } else { - Packet p = mat.template packet<Packet>(k); - visitor.initpacket(p, k); - k = PacketSize; + EIGEN_IF_CONSTEXPR (!AlreadyInitialized) { + if (size < PacketSize) { + visitor.init(mat.coeff(0), 0); + k = 1; + } else { + Packet p = mat.template packet<Packet>(k); + visitor.initpacket(p, k); + k = PacketSize; + } + if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return; } - if EIGEN_PREDICT_FALSE (short_circuit::run(visitor)) return; for (; k < packetEnd; k += PacketSize) { Packet p = mat.template packet<Packet>(k); visitor.packet(p, k); @@ -359,8 +371,9 @@ static constexpr int OuterSizeAtCompileTime = IsRowMajor ? RowsAtCompileTime : ColsAtCompileTime; // Linear packets can make an early scalar short-circuit exit more expensive. - static constexpr bool LinearAccess = - !ShortCircuitEvaluation && Evaluator::LinearAccess && visitor_has_linear_access<Visitor>::value; + // Preinitialized visitors opt into starting directly with packet traversal. + static constexpr bool LinearAccess = (!ShortCircuitEvaluation || visitor_already_initialized<Visitor>::value) && + Evaluator::LinearAccess && visitor_has_linear_access<Visitor>::value; static constexpr bool Vectorize = Evaluator::PacketAccess && static_cast<bool>(functor_traits<Visitor>::PacketAccess); static constexpr int PacketSize = packet_traits<Scalar>::size; @@ -414,6 +427,89 @@ namespace internal { +template <typename Scalar, bool Approximate> +struct fuzzy_constant_visitor { + Scalar value; + typename NumTraits<Scalar>::Real precision; + bool result = true; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(const Scalar& x, Index, Index = 0) { + result = result && (Approximate ? internal::isApprox(x, value, precision) + : internal::isMuchSmallerThan(x, Scalar(1), precision)); + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void init(const Scalar& x, Index r, Index c = 0) { (*this)(x, r, c); } + EIGEN_DEVICE_FUNC bool done() const { return !result; } + template <typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void packet(const Packet& x, Index, Index = 0) { + const Packet p = pset1<Packet>(precision); + Packet mask; + EIGEN_IF_CONSTEXPR (Approximate) { + const Packet v = pset1<Packet>(value); + mask = pcmp_le(pabs(psub(x, v)), pmul(pmin(pabs(x), pabs(v)), p)); + } else { + mask = pcmp_le(pabs(x), p); + } + // Reduce the comparison mask directly, without comparing its lanes to zero again. + result = result && !predux_any(pandnot(ptrue(mask), mask)); + } + template <typename Packet> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void initpacket(const Packet& x, Index r, Index c = 0) { + packet(x, r, c); + } +}; + +template <typename Scalar, bool Approximate> +struct functor_traits<fuzzy_constant_visitor<Scalar, Approximate>> { + static constexpr bool AlreadyInitialized = true; + static constexpr bool LinearAccess = true; + static constexpr int Cost = 4 * NumTraits<Scalar>::AddCost + NumTraits<Scalar>::MulCost; + // ARMv7 NEON flushes subnormal float operands and results; scalar VFP does not. + // Explicitly flushing would change exact-zero checks; normal operands can also have a subnormal difference. + static constexpr bool PacketAccess = + (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value) && + !(EIGEN_ARCH_ARM && std::is_same<Scalar, float>::value) && packet_traits<Scalar>::HasAbs && + packet_traits<Scalar>::HasCmp && + (!Approximate || + (packet_traits<Scalar>::HasSub && packet_traits<Scalar>::HasMin && packet_traits<Scalar>::HasMul)); +}; + +template <typename Scalar> +using use_fuzzy_constant_visitor = + bool_constant<std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value>; + +template <bool Approximate, typename Derived, + std::enable_if_t<use_fuzzy_constant_visitor<typename Derived::Scalar>::value, int> = 0> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool fuzzy_constant_all(const Derived& matrix, + const typename Derived::Scalar& value, + const typename Derived::RealScalar& precision) { + fuzzy_constant_visitor<typename Derived::Scalar, Approximate> visitor{value, precision}; + visit_impl<Derived, decltype(visitor), true>::run(matrix, visitor); + return visitor.result; +} + +// Separate overloads keep custom scalars from instantiating unused comparisons in C++14. +template <bool Approximate, typename Derived, + std::enable_if_t<!use_fuzzy_constant_visitor<typename Derived::Scalar>::value && Approximate, int> = 0> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool fuzzy_constant_all(const Derived& matrix, + const typename Derived::Scalar& value, + const typename Derived::RealScalar& precision) { + for (Index j = 0; j < matrix.cols(); ++j) + for (Index i = 0; i < matrix.rows(); ++i) + if (!internal::isApprox(matrix.coeff(i, j), value, precision)) return false; + return true; +} + +template <bool Approximate, typename Derived, + std::enable_if_t<!use_fuzzy_constant_visitor<typename Derived::Scalar>::value && !Approximate, int> = 0> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool fuzzy_constant_all(const Derived& matrix, const typename Derived::Scalar&, + const typename Derived::RealScalar& precision) { + using Scalar = typename Derived::Scalar; + for (Index j = 0; j < matrix.cols(); ++j) + for (Index i = 0; i < matrix.rows(); ++i) + if (!internal::isMuchSmallerThan(matrix.coeff(i, j), Scalar(1), precision)) return false; + return true; +} + template <typename Scalar> struct all_visitor { using result_type = bool; @@ -491,6 +587,36 @@ }; }; +// Reduces pisfinite masks directly: their lanes are all-ones or zero, so the coefficients are all finite exactly when +// no lane of the complement is set. predux_all, which all() on isFiniteTyped() uses, compares each lane against zero +// instead, which a compiler can only fold away when it can prove the operand is a mask. +template <typename Scalar> +struct all_finite_visitor { + using result_type = bool; + using Packet = typename packet_traits<Scalar>::type; + EIGEN_DEVICE_FUNC inline bool finite_predux(const Packet& p) const { return !predux_any(pnot(pisfinite(p))); } + EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index, Index) { res = (numext::isfinite)(value); } + EIGEN_DEVICE_FUNC inline void init(const Scalar& value, Index) { res = (numext::isfinite)(value); } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index, Index) { res = finite_predux(p); } + EIGEN_DEVICE_FUNC inline void initpacket(const Packet& p, Index) { res = finite_predux(p); } + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index, Index) { + res = res && (numext::isfinite)(value); + } + EIGEN_DEVICE_FUNC inline void operator()(const Scalar& value, Index) { res = res && (numext::isfinite)(value); } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index, Index) { res = res && finite_predux(p); } + EIGEN_DEVICE_FUNC inline void packet(const Packet& p, Index) { res = res && finite_predux(p); } + EIGEN_DEVICE_FUNC inline bool done() const { return !res; } + bool res = true; +}; +template <typename Scalar> +struct functor_traits<all_finite_visitor<Scalar>> { + enum { + Cost = NumTraits<Scalar>::ReadCost + NumTraits<Scalar>::MulCost, + LinearAccess = true, + PacketAccess = packet_traits<Scalar>::HasCmp + }; +}; + template <typename Derived, bool AlwaysTrue = NumTraits<typename traits<Derived>::Scalar>::IsInteger> struct all_finite_impl { static EIGEN_DEVICE_FUNC inline bool run(const Derived& /*derived*/) { return true; } @@ -498,7 +624,12 @@ #if !defined(__FINITE_MATH_ONLY__) || !(__FINITE_MATH_ONLY__) template <typename Derived> struct all_finite_impl<Derived, false> { - static EIGEN_DEVICE_FUNC inline bool run(const Derived& derived) { return derived.array().isFiniteTyped().all(); } + static EIGEN_DEVICE_FUNC inline bool run(const Derived& derived) { + using Visitor = all_finite_visitor<typename traits<Derived>::Scalar>; + Visitor visitor; + visit_impl<Derived, Visitor, /*ShortCircuitEvaluation*/ true>::run(derived, visitor); + return visitor.res; + } }; #endif @@ -560,6 +691,28 @@ return internal::all_finite_impl<Derived>::run(derived()); } +/** \returns true if all coefficients in this matrix are approximately equal to \a val, to within precision \a prec */ +template <typename Derived> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool DenseBase<Derived>::isApproxToConstant(const Scalar& val, + const RealScalar& prec) const { + typename internal::nested_eval<Derived, 1>::type self(derived()); + return internal::fuzzy_constant_all<true>(self, val, prec); +} + +/** \returns true if *this is approximately equal to the zero matrix, + * within the precision given by \a prec. + * + * Example: \include MatrixBase_isZero.cpp + * Output: \verbinclude MatrixBase_isZero.out + * + * \sa class CwiseNullaryOp, Zero() + */ +template <typename Derived> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool DenseBase<Derived>::isZero(const RealScalar& prec) const { + typename internal::nested_eval<Derived, 1>::type self(derived()); + return internal::fuzzy_constant_all<false>(self, Scalar(0), prec); +} + } // end namespace Eigen #endif // EIGEN_VISITOR_H
diff --git a/Eigen/src/Core/arch/AVX/Complex.h b/Eigen/src/Core/arch/AVX/Complex.h index a28621b..ce6b0ce 100644 --- a/Eigen/src/Core/arch/AVX/Complex.h +++ b/Eigen/src/Core/arch/AVX/Complex.h
@@ -527,8 +527,6 @@ /*---------------- load/store segment support ----------------*/ -/*---------------- std::complex<float> ----------------*/ - template <> struct has_packet_segment<Packet2cf> : std::true_type {}; @@ -536,6 +534,14 @@ struct has_packet_segment<Packet4cf> : std::true_type {}; template <> +struct has_packet_segment<Packet2cd> : std::true_type {}; + +// With AVX-512VL these packets use k-masked loads and stores instead; see AVX512/Complex.h. +#ifndef EIGEN_VECTORIZE_AVX512VL + +/*---------------- std::complex<float> ----------------*/ + +template <> inline Packet2cf ploaduSegment<Packet2cf>(const std::complex<float>* from, Index begin, Index count) { return (Packet2cf)_mm_maskload_ps(&numext::real_ref(*from), segment_mask_2x64(begin, count)); } @@ -560,9 +566,6 @@ /*---------------- std::complex<double> ----------------*/ template <> -struct has_packet_segment<Packet2cd> : std::true_type {}; - -template <> inline Packet2cd ploaduSegment<Packet2cd>(const std::complex<double>* from, Index begin, Index count) { return (Packet2cd)_mm256_maskload_pd(&numext::real_ref(*from), segment_mask_4x64(2 * begin, 2 * count)); } @@ -573,6 +576,8 @@ _mm256_maskstore_pd(&numext::real_ref(*to), segment_mask_4x64(2 * begin, 2 * count), from.v); } +#endif // EIGEN_VECTORIZE_AVX512VL + /*---------------- end load/store segment support ----------------*/ EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_POP
diff --git a/Eigen/src/Core/arch/AVX/MathFunctions.h b/Eigen/src/Core/arch/AVX/MathFunctions.h index db134f3..c43e194 100644 --- a/Eigen/src/Core/arch/AVX/MathFunctions.h +++ b/Eigen/src/Core/arch/AVX/MathFunctions.h
@@ -57,6 +57,14 @@ return _mm256_insertf128_pd(_mm256_castpd128_pd256(ptan(_mm256_castpd256_pd128(x))), ptan(_mm256_extractf128_pd(x, 1)), 1); } + +// AVX-only Packet4d has no integer packet. Classify the bits using the SSE halves +// so subnormals survive DAZ and NaNs survive fast-math. +template <> +EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet4d psign<Packet4d>(const Packet4d& x) { + return _mm256_insertf128_pd(_mm256_castpd128_pd256(psign(_mm256_castpd256_pd128(x))), + psign(_mm256_extractf128_pd(x, 1)), 1); +} #endif EIGEN_GENERIC_PACKET_FUNCTION(atan, Packet4d) EIGEN_GENERIC_PACKET_FUNCTION(exp2, Packet4d)
diff --git a/Eigen/src/Core/arch/AVX/PacketMath.h b/Eigen/src/Core/arch/AVX/PacketMath.h index 129c8e0..6035550 100644 --- a/Eigen/src/Core/arch/AVX/PacketMath.h +++ b/Eigen/src/Core/arch/AVX/PacketMath.h
@@ -1879,23 +1879,51 @@ return pfrexp_generic(a, exponent); } +#ifndef EIGEN_VECTORIZE_AVX2 +namespace unary_pow { +// The exponent-bit operations the integer-exponent pow needs on Packet4d, likewise without Packet4l: AVX has no +// 256-bit integer shift or subtract, but SSE2's 128-bit ones, present on every x86-64 target, suffice. Split the +// bit pattern into its two 128-bit halves, operate on each with SSE2, and reassemble. +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4d exponent_bits_shift_right<Packet4d>(const Packet4d& bits) { + __m256i i = _mm256_castpd_si256(bits); + __m128i lo = _mm_srli_epi64(_mm256_extractf128_si256(i, 0), 52); + __m128i hi = _mm_srli_epi64(_mm256_extractf128_si256(i, 1), 52); + return _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1)); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4d exponent_bits_sub<Packet4d>(const Packet4d& a_bits, + const Packet4d& b_bits) { + __m256i ai = _mm256_castpd_si256(a_bits); + __m256i bi = _mm256_castpd_si256(b_bits); + __m128i lo = _mm_sub_epi64(_mm256_extractf128_si256(ai, 0), _mm256_extractf128_si256(bi, 0)); + __m128i hi = _mm_sub_epi64(_mm256_extractf128_si256(ai, 1), _mm256_extractf128_si256(bi, 1)); + return _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1)); +} +template <> +struct has_exponent_bit_ops<Packet4d> : true_type {}; +} // namespace unary_pow +#endif + template <> EIGEN_STRONG_INLINE Packet8f pldexp<Packet8f>(const Packet8f& a, const Packet8f& exponent) { return pldexp_generic(a, exponent); } // Build 2^k as Packet4d from a Packet4i holding the biased int32 exponent in -// each lane. AVX2 has a single-instruction widen+shift path; AVX-only must -// split the 128-bit input into two halves, widen+shift each separately with -// SSE intrinsics, and reassemble with vinsertf128. +// each lane, shifted by Shift: 52 for the biased exponent, 51 for twice it. +// AVX2 has a single-instruction widen+shift path; AVX-only must split the +// 128-bit input into two halves, widen+shift each separately with SSE +// intrinsics, and reassemble with vinsertf128. +template <int Shift = 52> EIGEN_STRONG_INLINE Packet4d pldexp_avx_pow2_from_biased(const Packet4i& biased) { #ifdef EIGEN_VECTORIZE_AVX2 - return _mm256_castsi256_pd(_mm256_slli_epi64(_mm256_cvtepi32_epi64(biased), 52)); + return _mm256_castsi256_pd(_mm256_slli_epi64(_mm256_cvtepi32_epi64(biased), Shift)); #else __m128i lo = _mm_cvtepi32_epi64(biased); // SSE4.1: lower 2 int32 -> 2 int64 __m128i hi = _mm_cvtepi32_epi64(_mm_unpackhi_epi64(biased, biased)); // upper 2 int32 -> 2 int64 - lo = _mm_slli_epi64(lo, 52); - hi = _mm_slli_epi64(hi, 52); + lo = _mm_slli_epi64(lo, Shift); + hi = _mm_slli_epi64(hi, Shift); return _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1)); #endif } @@ -1906,14 +1934,14 @@ const Packet4d max_exponent = pset1<Packet4d>(2099.0); const Packet4i e = _mm256_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); - // Preserve the sequential 4-way split; see pldexp_generic. + // The single-rounding split of pldexp_generic. const Packet4i bias = pset1<Packet4i>(1023); - const Packet4i b = parithmetic_shift_right<2>(e); // floor(e/4) - const Packet4i b_remainder = psub(psub(e, b), padd(b, b)); // e - 3b (depth 2) - const Packet4d c1 = pldexp_avx_pow2_from_biased(padd(b, bias)); // 2^b - const Packet4d c2 = pldexp_avx_pow2_from_biased(padd(b_remainder, bias)); // 2^(e-3b) + const Packet4i b = pmin(pmax(e, pset1<Packet4i>(-1022)), pset1<Packet4i>(1022)); + const Packet4i t = pandnot(psub(e, b), pset1<Packet4i>(1)); // even + const Packet4d c1 = pldexp_avx_pow2_from_biased<51>(padd(t, padd(bias, bias))); // 2^(t/2) + const Packet4d c2 = pldexp_avx_pow2_from_biased(padd(psub(e, t), bias)); // 2^(e-t) - return pmul(pmul(pmul(pmul(a, c1), c1), c1), c2); // a * 2^e + return pldexp_apply_factors(a, c1, c2); // a * 2^e } template <> @@ -2672,6 +2700,15 @@ return _mm_cmpgt_epi16(_mm_and_si128(a, _mm_set1_epi16(kAbsMask)), _mm_set1_epi16(kInf)); } +// Compare encoded lanes: widening bf16 subnormals to float loses them under DAZ/FZ. +template <> +EIGEN_STRONG_INLINE Packet8bf psign<Packet8bf>(const Packet8bf& a) { + const __m128i magnitude = _mm_and_si128(a, _mm_set1_epi16(0x7fff)); + const __m128i is_nan = _mm_cmpgt_epi16(magnitude, _mm_set1_epi16(0x7f80)); + const __m128i keep = _mm_or_si128(is_nan, _mm_set1_epi16(static_cast<short>(0xbf80u))); + return _mm_sign_epi16(_mm_and_si128(_mm_or_si128(a, _mm_set1_epi16(0x3f80)), keep), magnitude); +} + template <> EIGEN_STRONG_INLINE Packet8bf pisfinite<Packet8bf>(const Packet8bf& a) { constexpr uint16_t kInf = ((1 << 8) - 1) << 7; @@ -2815,6 +2852,44 @@ /*---------------- load/store segment support ----------------*/ +template <> +struct has_packet_segment<Packet4f> : std::true_type {}; + +template <> +struct has_packet_segment<Packet8f> : std::true_type {}; + +template <> +struct has_packet_segment<Packet4i> : std::true_type {}; + +template <> +struct has_packet_segment<Packet8i> : std::true_type {}; + +template <> +struct has_packet_segment<Packet4ui> : std::true_type {}; + +template <> +struct has_packet_segment<Packet8ui> : std::true_type {}; + +template <> +struct has_packet_segment<Packet2d> : std::true_type {}; + +template <> +struct has_packet_segment<Packet4d> : std::true_type {}; + +#ifdef EIGEN_VECTORIZE_AVX2 +template <> +struct has_packet_segment<Packet2l> : std::true_type {}; + +template <> +struct has_packet_segment<Packet4l> : std::true_type {}; + +template <> +struct has_packet_segment<Packet4ul> : std::true_type {}; +#endif + +// With AVX-512VL these packets use k-masked loads and stores instead; see AVX512/PacketMath.h. +#ifndef EIGEN_VECTORIZE_AVX512VL + // returns a mask of 8-bit elements (at most 4) that are all 1's in the range [begin, begin + count) and 0 elsewhere. inline __m128i segment_mask_4x8(Index begin, Index count) { eigen_assert(begin >= 0 && begin + count <= 4); @@ -2886,12 +2961,6 @@ /*---------------- float ----------------*/ template <> -struct has_packet_segment<Packet4f> : std::true_type {}; - -template <> -struct has_packet_segment<Packet8f> : std::true_type {}; - -template <> inline Packet4f ploaduSegment<Packet4f>(const float* from, Index begin, Index count) { return _mm_maskload_ps(from, segment_mask_4x32(begin, count)); } @@ -2913,12 +2982,6 @@ /*---------------- int32 ----------------*/ -template <> -struct has_packet_segment<Packet4i> : std::true_type {}; - -template <> -struct has_packet_segment<Packet8i> : std::true_type {}; - #ifdef EIGEN_VECTORIZE_AVX2 template <> @@ -2968,12 +3031,6 @@ /*---------------- uint32 ----------------*/ template <> -struct has_packet_segment<Packet4ui> : std::true_type {}; - -template <> -struct has_packet_segment<Packet8ui> : std::true_type {}; - -template <> inline Packet4ui ploaduSegment<Packet4ui>(const uint32_t* from, Index begin, Index count) { return Packet4ui(ploaduSegment<Packet4i>(reinterpret_cast<const int*>(from), begin, count)); } @@ -2996,12 +3053,6 @@ /*---------------- double ----------------*/ template <> -struct has_packet_segment<Packet2d> : std::true_type {}; - -template <> -struct has_packet_segment<Packet4d> : std::true_type {}; - -template <> inline Packet2d ploaduSegment<Packet2d>(const double* from, Index begin, Index count) { return _mm_maskload_pd(from, segment_mask_2x64(begin, count)); } @@ -3026,12 +3077,6 @@ /*---------------- int64_t ----------------*/ template <> -struct has_packet_segment<Packet2l> : std::true_type {}; - -template <> -struct has_packet_segment<Packet4l> : std::true_type {}; - -template <> inline Packet2l ploaduSegment<Packet2l>(const int64_t* from, Index begin, Index count) { return _mm_maskload_epi64(reinterpret_cast<const long long*>(from), segment_mask_2x64(begin, count)); } @@ -3051,9 +3096,6 @@ /*---------------- uint64_t ----------------*/ template <> -struct has_packet_segment<Packet4ul> : std::true_type {}; - -template <> inline Packet4ul ploaduSegment<Packet4ul>(const uint64_t* from, Index begin, Index count) { return Packet4ul(ploaduSegment<Packet4l>(reinterpret_cast<const int64_t*>(from), begin, count)); } @@ -3063,6 +3105,8 @@ } #endif +#endif // EIGEN_VECTORIZE_AVX512VL + /*---------------- end load/store segment support ----------------*/ } // end namespace internal
diff --git a/Eigen/src/Core/arch/AVX512/Complex.h b/Eigen/src/Core/arch/AVX512/Complex.h index 2cb3feb..b3d90e2 100644 --- a/Eigen/src/Core/arch/AVX512/Complex.h +++ b/Eigen/src/Core/arch/AVX512/Complex.h
@@ -449,6 +449,88 @@ EIGEN_INSTANTIATE_COMPLEX_MATH_FUNCS(Packet4cd) EIGEN_INSTANTIATE_COMPLEX_MATH_FUNCS(Packet8cf) +/*---------------- load/store segment support ----------------*/ + +// A complex lane is two real lanes, so the k-mask is formed over 2 * begin and 2 * count. + +/*---------------- std::complex<float> ----------------*/ + +template <> +struct has_packet_segment<Packet8cf> : std::true_type {}; + +template <> +inline Packet8cf ploaduSegment<Packet8cf>(const std::complex<float>* from, Index begin, Index count) { + return Packet8cf( + _mm512_maskz_loadu_ps(static_cast<__mmask16>(segment_kmask(2 * begin, 2 * count)), &numext::real_ref(*from))); +} + +template <> +inline void pstoreuSegment<std::complex<float>, Packet8cf>(std::complex<float>* to, const Packet8cf& from, Index begin, + Index count) { + _mm512_mask_storeu_ps(&numext::real_ref(*to), static_cast<__mmask16>(segment_kmask(2 * begin, 2 * count)), from.v); +} + +/*---------------- std::complex<double> ----------------*/ + +template <> +struct has_packet_segment<Packet4cd> : std::true_type {}; + +template <> +inline Packet4cd ploaduSegment<Packet4cd>(const std::complex<double>* from, Index begin, Index count) { + return Packet4cd( + _mm512_maskz_loadu_pd(static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), &numext::real_ref(*from))); +} + +template <> +inline void pstoreuSegment<std::complex<double>, Packet4cd>(std::complex<double>* to, const Packet4cd& from, + Index begin, Index count) { + _mm512_mask_storeu_pd(&numext::real_ref(*to), static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), from.v); +} + +#ifdef EIGEN_VECTORIZE_AVX512VL + +// The 256- and 128-bit packets declare has_packet_segment in AVX/Complex.h. + +template <> +inline Packet4cf ploaduSegment<Packet4cf>(const std::complex<float>* from, Index begin, Index count) { + return Packet4cf( + _mm256_maskz_loadu_ps(static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), &numext::real_ref(*from))); +} + +template <> +inline void pstoreuSegment<std::complex<float>, Packet4cf>(std::complex<float>* to, const Packet4cf& from, Index begin, + Index count) { + _mm256_mask_storeu_ps(&numext::real_ref(*to), static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), from.v); +} + +template <> +inline Packet2cf ploaduSegment<Packet2cf>(const std::complex<float>* from, Index begin, Index count) { + return Packet2cf( + _mm_maskz_loadu_ps(static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), &numext::real_ref(*from))); +} + +template <> +inline void pstoreuSegment<std::complex<float>, Packet2cf>(std::complex<float>* to, const Packet2cf& from, Index begin, + Index count) { + _mm_mask_storeu_ps(&numext::real_ref(*to), static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), from.v); +} + +template <> +inline Packet2cd ploaduSegment<Packet2cd>(const std::complex<double>* from, Index begin, Index count) { + return Packet2cd( + _mm256_maskz_loadu_pd(static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), &numext::real_ref(*from))); +} + +template <> +inline void pstoreuSegment<std::complex<double>, Packet2cd>(std::complex<double>* to, const Packet2cd& from, + Index begin, Index count) { + _mm256_mask_storeu_pd(&numext::real_ref(*to), static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), from.v); +} + +#endif // EIGEN_VECTORIZE_AVX512VL + +/*---------------- end load/store segment support ----------------*/ + EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_POP } // end namespace internal
diff --git a/Eigen/src/Core/arch/AVX512/GemmKernel.h b/Eigen/src/Core/arch/AVX512/GemmKernel.h index 2c039eb..316a74f 100644 --- a/Eigen/src/Core/arch/AVX512/GemmKernel.h +++ b/Eigen/src/Core/arch/AVX512/GemmKernel.h
@@ -627,7 +627,7 @@ load_a_impl<um, uk, nelems, ktail>(std::make_integer_sequence<int, um_vecs - um>{}, ao); } - template <int uk, int pow, int count, int um_vecs, int b_unroll, bool ktail, bool fetch_x> + template <int uk, int pow, int count, int um_vecs, int b_unroll, bool ktail, bool fetch_x, bool preload_next_k = true> EIGEN_ALWAYS_INLINE void innerkernel_1pow_one(const Scalar*& aa, const Scalar* const& ao, const Scalar* const& bo, int& fetchA_idx, int& fetchB_idx) { const int idx = (pow / 2) + count; @@ -644,27 +644,33 @@ EIGEN_IF_CONSTEXPR (b_unroll >= pow) { compute<0, um_vecs, idx, uk, fetch_x, ktail>(ao, bo, fetchA_idx, fetchB_idx, b_reg); - const Scalar* b_addr = bo + b_unroll * uk + idx + 1 + (b_unroll > 1) * !use_less_b_regs - b_shift; - b_load(b_reg, b_addr); + constexpr int b_load_offset = idx + 1 + (b_unroll > 1) * !use_less_b_regs; + EIGEN_IF_CONSTEXPR (preload_next_k || b_load_offset < b_unroll) { + const Scalar* b_addr = bo + b_unroll * uk + b_load_offset - b_shift; + b_load(b_reg, b_addr); + } } } - template <int uk, int pow, int count, int um_vecs, int b_unroll, bool ktail, bool fetch_x, int... indices> + template <int uk, int pow, int count, int um_vecs, int b_unroll, bool ktail, bool fetch_x, bool preload_next_k, + int... indices> EIGEN_ALWAYS_INLINE void innerkernel_1pow_impl(std::integer_sequence<int, indices...>, const Scalar*& aa, const Scalar* const& ao, const Scalar* const& bo, int& fetchA_idx, int& fetchB_idx) { - int unused[] = {0, (innerkernel_1pow_one<uk, pow, count + indices, um_vecs, b_unroll, ktail, fetch_x>( - aa, ao, bo, fetchA_idx, fetchB_idx), - 0)...}; + int unused[] = {0, + (innerkernel_1pow_one<uk, pow, count + indices, um_vecs, b_unroll, ktail, fetch_x, preload_next_k>( + aa, ao, bo, fetchA_idx, fetchB_idx), + 0)...}; EIGEN_UNUSED_VARIABLE(unused); } - template <int uk, int pow, int count, int um_vecs, int b_unroll, bool ktail, bool fetch_x, bool c_fetch> + template <int uk, int pow, int count, int um_vecs, int b_unroll, bool ktail, bool fetch_x, bool c_fetch, + bool preload_next_k = true> EIGEN_ALWAYS_INLINE void innerkernel_1pow(const Scalar*& aa, const Scalar* const& ao, const Scalar* const& bo, Scalar*& co2, int& fetchA_idx, int& fetchB_idx) { constexpr int max_count = (pow + 1) / 2; static_assert(count <= max_count, "invalid B load range"); - innerkernel_1pow_impl<uk, pow, count, um_vecs, b_unroll, ktail, fetch_x>( + innerkernel_1pow_impl<uk, pow, count, um_vecs, b_unroll, ktail, fetch_x, preload_next_k>( std::make_integer_sequence<int, max_count - count>{}, aa, ao, bo, fetchA_idx, fetchB_idx); // Maybe prefetch C data after count-loop. @@ -678,22 +684,26 @@ } template <int uk, int max_b_unroll, int a_unroll, int b_unroll, bool ktail, bool fetch_x, bool c_fetch, - bool no_a_preload = false> + bool preload_next_k = true> EIGEN_ALWAYS_INLINE void innerkernel_1uk(const Scalar*& aa, const Scalar* const& ao, const Scalar* const& bo, Scalar*& co2, int& fetchA_idx, int& fetchB_idx) { const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); EIGEN_IF_CONSTEXPR (max_b_unroll >= 1) - innerkernel_1pow<uk, 1, 0, um_vecs, b_unroll, ktail, fetch_x, c_fetch>(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + innerkernel_1pow<uk, 1, 0, um_vecs, b_unroll, ktail, fetch_x, c_fetch, preload_next_k>(aa, ao, bo, co2, + fetchA_idx, fetchB_idx); EIGEN_IF_CONSTEXPR (max_b_unroll >= 2) - innerkernel_1pow<uk, 2, 0, um_vecs, b_unroll, ktail, fetch_x, c_fetch>(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + innerkernel_1pow<uk, 2, 0, um_vecs, b_unroll, ktail, fetch_x, c_fetch, preload_next_k>(aa, ao, bo, co2, + fetchA_idx, fetchB_idx); EIGEN_IF_CONSTEXPR (max_b_unroll >= 4) - innerkernel_1pow<uk, 4, 0, um_vecs, b_unroll, ktail, fetch_x, c_fetch>(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + innerkernel_1pow<uk, 4, 0, um_vecs, b_unroll, ktail, fetch_x, c_fetch, preload_next_k>(aa, ao, bo, co2, + fetchA_idx, fetchB_idx); EIGEN_IF_CONSTEXPR (max_b_unroll >= 8) - innerkernel_1pow<uk, 8, 0, um_vecs, b_unroll, ktail, fetch_x, c_fetch>(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + innerkernel_1pow<uk, 8, 0, um_vecs, b_unroll, ktail, fetch_x, c_fetch, preload_next_k>(aa, ao, bo, co2, + fetchA_idx, fetchB_idx); // Load A after pow-loop. Skip this at the end to prevent running over the buffer - if (!no_a_preload) load_a<0, um_vecs, uk, a_unroll, ktail>(ao); + if (preload_next_k) load_a<0, um_vecs, uk, a_unroll, ktail>(ao); } /* Inner kernel loop structure. @@ -710,8 +720,9 @@ * if (b_unroll >= pow) { * compute<0, um_vecs, idx, uk, fetchx, ktail>(ao, bo, fetchA_idx, fetchB_idx, b_reg); * - * const Scalar *b_addr = bo + b_unroll * uk + idx + 1 + (b_unroll > 1) - b_shift ; - * b_load(b_reg, b_addr); + * constexpr int b_load_offset = idx + 1 + (b_unroll > 1) * !use_less_b_regs; + * if (preload_next_k || b_load_offset < b_unroll) + * b_load(b_reg, bo + b_unroll * uk + b_load_offset - b_shift); * } * idx++; * } @@ -727,7 +738,7 @@ * } * * Load A. - * load_a<0, um_vecs, uk, ktail, a_unroll>(ao); + * if (preload_next_k) load_a<0, um_vecs, uk, a_unroll, ktail>(ao); * } * * Advance A/B pointers after uk-loop. @@ -736,7 +747,7 @@ */ template <int a_unroll, int b_unroll, int k_factor, int max_b_unroll, int max_k_factor, bool c_fetch, - bool no_a_preload = false> + bool preload_next_k = true> EIGEN_ALWAYS_INLINE void innerkernel(const Scalar*& aa, const Scalar*& ao, const Scalar*& bo, Scalar*& co2) { int fetchA_idx = 0; int fetchB_idx = 0; @@ -745,20 +756,19 @@ const bool ktail = k_factor == 1; static_assert(k_factor <= 4 && k_factor > 0, "innerkernel maximum k_factor supported is 4"); - static_assert(no_a_preload == false || (no_a_preload == true && k_factor == 1), - "skipping a preload only allowed when k unroll is 1"); + static_assert(preload_next_k || k_factor == 1, "skipping next-k preload only allowed when k unroll is 1"); if (k_factor > 0) - innerkernel_1uk<0, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>( + innerkernel_1uk<0, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, preload_next_k>( aa, ao, bo, co2, fetchA_idx, fetchB_idx); if (k_factor > 1) - innerkernel_1uk<1, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>( + innerkernel_1uk<1, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, preload_next_k>( aa, ao, bo, co2, fetchA_idx, fetchB_idx); if (k_factor > 2) - innerkernel_1uk<2, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>( + innerkernel_1uk<2, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, preload_next_k>( aa, ao, bo, co2, fetchA_idx, fetchB_idx); if (k_factor > 3) - innerkernel_1uk<3, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>( + innerkernel_1uk<3, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, preload_next_k>( aa, ao, bo, co2, fetchA_idx, fetchB_idx); // Advance A/B pointers after uk-loop. @@ -779,7 +789,7 @@ } b_load(zmm[b_regs[0]], bo - b_shift + 0); - EIGEN_IF_CONSTEXPR (!use_less_b_regs) { + EIGEN_IF_CONSTEXPR (b_unroll > 1 && !use_less_b_regs) { b_load(zmm[b_regs[1]], bo - b_shift + 1); } @@ -829,7 +839,7 @@ loop_count--; } if (loop_count > 0) { - innerkernel<a_unroll, b_unroll, 1, max_b_unroll, max_k_factor, 0, true>(aa, ao, bo, co2); + innerkernel<a_unroll, b_unroll, 1, max_b_unroll, max_k_factor, 0, false>(aa, ao, bo, co2); } // Update C matrix. @@ -999,6 +1009,7 @@ EIGEN_DONT_INLINE void gemm_kern_avx512(Index m, Index n, Index k, Scalar* alpha, const Scalar* a, const Scalar* b, Scalar* c, Index ldc, Index inc = 1, Index a_stride = -1, Index b_stride = -1, Index a_off = 0, Index b_off = 0) { + if (m <= 0 || n <= 0 || k <= 0) return; if (a_stride == -1) a_stride = k; if (b_stride == -1) b_stride = k;
diff --git a/Eigen/src/Core/arch/AVX512/MathFunctionsFP16.h b/Eigen/src/Core/arch/AVX512/MathFunctionsFP16.h index 48115c5..d4f345b 100644 --- a/Eigen/src/Core/arch/AVX512/MathFunctionsFP16.h +++ b/Eigen/src/Core/arch/AVX512/MathFunctionsFP16.h
@@ -67,7 +67,10 @@ // pldexp template <> EIGEN_STRONG_INLINE Packet32h pldexp<Packet32h>(const Packet32h& a, const Packet32h& exponent) { - return pldexp_generic(a, exponent); + // Clamped for the reasons at pldexp<Packet16f>; an int exponent beyond the range of half arrives as +-inf. + const Packet32h max_exponent = pset1<Packet32h>(Eigen::half(41.0f)); + const Packet32h e = pmin(pmax(exponent, pnegate(max_exponent)), max_exponent); + return _mm512_scalef_ph(a, _mm512_roundscale_ph(e, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)); } } // end namespace internal
diff --git a/Eigen/src/Core/arch/AVX512/PacketMath.h b/Eigen/src/Core/arch/AVX512/PacketMath.h index 2596313..7522950 100644 --- a/Eigen/src/Core/arch/AVX512/PacketMath.h +++ b/Eigen/src/Core/arch/AVX512/PacketMath.h
@@ -1478,26 +1478,30 @@ template <> EIGEN_STRONG_INLINE Packet16f pldexp<Packet16f>(const Packet16f& a, const Packet16f& exponent) { - return pldexp_generic(a, exponent); + // vscalef is a * 2^floor(exponent), rounded once, and pldexp takes (int)exponent: truncate first. Clamp as + // pldexp_generic does: vscalef gives NaN for 0 * 2^inf and inf * 2^-inf, and a number for NaN * 2^(+-inf), and an + // int exponent beyond the range of half is infinite by the time it reaches here through half2float. + const Packet16f max_exponent = pset1<Packet16f>(278.0f); + const Packet16f e = pmin(pmax(exponent, pnegate(max_exponent)), max_exponent); + return _mm512_scalef_ps(a, _mm512_roundscale_ps(e, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)); } template <> EIGEN_STRONG_INLINE Packet8d pldexp<Packet8d>(const Packet8d& a, const Packet8d& exponent) { - // Clamp exponent to [-2099, 2099] - const Packet8d max_exponent = pset1<Packet8d>(2099.0); - const Packet8i e = _mm512_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); + const Packet8d max_exponent = pset1<Packet8d>(2099.0); // see pldexp<Packet16f> + const Packet8d e = pmin(pmax(exponent, pnegate(max_exponent)), max_exponent); + return _mm512_scalef_pd(a, _mm512_roundscale_pd(e, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)); +} - // Preserve the sequential 4-way split; see pldexp_generic. - // 2^b and 2^(e-3b) are built by widening the biased int32 exponent to int64 - // with vpmovsxdq and shifting into the double exponent field with vpsllq. - const Packet8i bias = pset1<Packet8i>(1023); - const Packet8i b = parithmetic_shift_right<2>(e); // floor(e/4) - const Packet8i b_remainder = psub(psub(e, b), padd(b, b)); // e - 3b (depth 2) - const Packet8d c1 = _mm512_castsi512_pd(_mm512_slli_epi64(_mm512_cvtepi32_epi64(padd(b, bias)), 52)); // 2^b - const Packet8d c2 = - _mm512_castsi512_pd(_mm512_slli_epi64(_mm512_cvtepi32_epi64(padd(b_remainder, bias)), 52)); // 2^(e-3b) +// pldexp_fast's callers pass integral exponents, and vscalef floors, so neither the clamp nor the truncation. +template <> +EIGEN_STRONG_INLINE Packet16f pldexp_fast<Packet16f>(const Packet16f& a, const Packet16f& exponent) { + return _mm512_scalef_ps(a, exponent); +} - return pmul(pmul(pmul(pmul(a, c1), c1), c1), c2); // a * 2^e +template <> +EIGEN_STRONG_INLINE Packet8d pldexp_fast<Packet8d>(const Packet8d& a, const Packet8d& exponent) { + return _mm512_scalef_pd(a, exponent); } #ifdef EIGEN_VECTORIZE_AVX512DQ @@ -2782,6 +2786,15 @@ return _mm256_cmpgt_epi16(_mm256_and_si256(a, _mm256_set1_epi16(kAbsMask)), _mm256_set1_epi16(kInf)); } +// Compare encoded lanes: widening bf16 subnormals to float loses them under DAZ/FZ. +template <> +EIGEN_STRONG_INLINE Packet16bf psign<Packet16bf>(const Packet16bf& a) { + const __m256i magnitude = _mm256_and_si256(a, _mm256_set1_epi16(0x7fff)); + const __m256i is_nan = _mm256_cmpgt_epi16(magnitude, _mm256_set1_epi16(0x7f80)); + const __m256i keep = _mm256_or_si256(is_nan, _mm256_set1_epi16(static_cast<short>(0xbf80u))); + return _mm256_sign_epi16(_mm256_and_si256(_mm256_or_si256(a, _mm256_set1_epi16(0x3f80)), keep), magnitude); +} + template <> EIGEN_STRONG_INLINE Packet16bf pisfinite<Packet16bf>(const Packet16bf& a) { constexpr uint16_t kInf = ((1 << 8) - 1) << 7; @@ -3185,6 +3198,270 @@ return _mm_srli_epi16(a, N); } +/*---------------- load/store segment support ----------------*/ + +// Returns a k-mask with bits [begin, begin + count) set. 64-bit arithmetic lets count reach the lane +// count of the widest AVX-512 packet (32) without shifting a 1 past the top bit. +EIGEN_STRONG_INLINE uint64_t segment_kmask(Index begin, Index count) { + eigen_assert(begin >= 0 && count >= 0 && begin + count <= 32); + return ((uint64_t(1) << count) - 1) << begin; +} + +/*---------------- float ----------------*/ + +template <> +struct has_packet_segment<Packet16f> : std::true_type {}; + +template <> +inline Packet16f ploaduSegment<Packet16f>(const float* from, Index begin, Index count) { + return _mm512_maskz_loadu_ps(static_cast<__mmask16>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<float, Packet16f>(float* to, const Packet16f& from, Index begin, Index count) { + _mm512_mask_storeu_ps(to, static_cast<__mmask16>(segment_kmask(begin, count)), from); +} + +/*---------------- double ----------------*/ + +template <> +struct has_packet_segment<Packet8d> : std::true_type {}; + +template <> +inline Packet8d ploaduSegment<Packet8d>(const double* from, Index begin, Index count) { + return _mm512_maskz_loadu_pd(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<double, Packet8d>(double* to, const Packet8d& from, Index begin, Index count) { + _mm512_mask_storeu_pd(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +/*---------------- int32 ----------------*/ + +template <> +struct has_packet_segment<Packet16i> : std::true_type {}; + +template <> +inline Packet16i ploaduSegment<Packet16i>(const int* from, Index begin, Index count) { + return _mm512_maskz_loadu_epi32(static_cast<__mmask16>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<int, Packet16i>(int* to, const Packet16i& from, Index begin, Index count) { + _mm512_mask_storeu_epi32(to, static_cast<__mmask16>(segment_kmask(begin, count)), from); +} + +/*---------------- int64_t ----------------*/ + +template <> +struct has_packet_segment<Packet8l> : std::true_type {}; + +template <> +inline Packet8l ploaduSegment<Packet8l>(const int64_t* from, Index begin, Index count) { + return _mm512_maskz_loadu_epi64(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<int64_t, Packet8l>(int64_t* to, const Packet8l& from, Index begin, Index count) { + _mm512_mask_storeu_epi64(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +#ifdef EIGEN_VECTORIZE_AVX512VL + +// The 256- and 128-bit packets declare has_packet_segment in AVX/PacketMath.h; with AVX-512VL their +// segments are k-masked as well, which avoids vmaskmov and the vector mask it needs. + +/*---------------- float ----------------*/ + +template <> +inline Packet8f ploaduSegment<Packet8f>(const float* from, Index begin, Index count) { + return _mm256_maskz_loadu_ps(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<float, Packet8f>(float* to, const Packet8f& from, Index begin, Index count) { + _mm256_mask_storeu_ps(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline Packet4f ploaduSegment<Packet4f>(const float* from, Index begin, Index count) { + return _mm_maskz_loadu_ps(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<float, Packet4f>(float* to, const Packet4f& from, Index begin, Index count) { + _mm_mask_storeu_ps(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +/*---------------- double ----------------*/ + +template <> +inline Packet4d ploaduSegment<Packet4d>(const double* from, Index begin, Index count) { + return _mm256_maskz_loadu_pd(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<double, Packet4d>(double* to, const Packet4d& from, Index begin, Index count) { + _mm256_mask_storeu_pd(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline Packet2d ploaduSegment<Packet2d>(const double* from, Index begin, Index count) { + return _mm_maskz_loadu_pd(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<double, Packet2d>(double* to, const Packet2d& from, Index begin, Index count) { + _mm_mask_storeu_pd(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +/*---------------- int32 ----------------*/ + +template <> +inline Packet8i ploaduSegment<Packet8i>(const int* from, Index begin, Index count) { + return _mm256_maskz_loadu_epi32(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<int, Packet8i>(int* to, const Packet8i& from, Index begin, Index count) { + _mm256_mask_storeu_epi32(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline Packet4i ploaduSegment<Packet4i>(const int* from, Index begin, Index count) { + return _mm_maskz_loadu_epi32(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<int, Packet4i>(int* to, const Packet4i& from, Index begin, Index count) { + _mm_mask_storeu_epi32(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +/*---------------- uint32 ----------------*/ + +template <> +inline Packet8ui ploaduSegment<Packet8ui>(const uint32_t* from, Index begin, Index count) { + return Packet8ui(ploaduSegment<Packet8i>(reinterpret_cast<const int*>(from), begin, count)); +} + +template <> +inline void pstoreuSegment<uint32_t, Packet8ui>(uint32_t* to, const Packet8ui& from, Index begin, Index count) { + pstoreuSegment<int, Packet8i>(reinterpret_cast<int*>(to), Packet8i(from), begin, count); +} + +template <> +inline Packet4ui ploaduSegment<Packet4ui>(const uint32_t* from, Index begin, Index count) { + return Packet4ui(ploaduSegment<Packet4i>(reinterpret_cast<const int*>(from), begin, count)); +} + +template <> +inline void pstoreuSegment<uint32_t, Packet4ui>(uint32_t* to, const Packet4ui& from, Index begin, Index count) { + pstoreuSegment<int, Packet4i>(reinterpret_cast<int*>(to), Packet4i(from), begin, count); +} + +/*---------------- int64_t ----------------*/ + +template <> +inline Packet4l ploaduSegment<Packet4l>(const int64_t* from, Index begin, Index count) { + return _mm256_maskz_loadu_epi64(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<int64_t, Packet4l>(int64_t* to, const Packet4l& from, Index begin, Index count) { + _mm256_mask_storeu_epi64(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline Packet2l ploaduSegment<Packet2l>(const int64_t* from, Index begin, Index count) { + return _mm_maskz_loadu_epi64(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<int64_t, Packet2l>(int64_t* to, const Packet2l& from, Index begin, Index count) { + _mm_mask_storeu_epi64(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +/*---------------- uint64_t ----------------*/ + +template <> +inline Packet4ul ploaduSegment<Packet4ul>(const uint64_t* from, Index begin, Index count) { + return Packet4ul(ploaduSegment<Packet4l>(reinterpret_cast<const int64_t*>(from), begin, count)); +} + +template <> +inline void pstoreuSegment<uint64_t, Packet4ul>(uint64_t* to, const Packet4ul& from, Index begin, Index count) { + pstoreuSegment<int64_t, Packet4l>(reinterpret_cast<int64_t*>(to), Packet4l(from), begin, count); +} + +#if defined(__AVX512BW__) + +// 16-bit lanes need the AVX-512BW word-granular masked moves. + +/*---------------- bfloat16 ----------------*/ + +template <> +struct has_packet_segment<Packet16bf> : std::true_type {}; + +template <> +struct has_packet_segment<Packet8bf> : std::true_type {}; + +template <> +inline Packet16bf ploaduSegment<Packet16bf>(const bfloat16* from, Index begin, Index count) { + return _mm256_maskz_loadu_epi16(static_cast<__mmask16>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<bfloat16, Packet16bf>(bfloat16* to, const Packet16bf& from, Index begin, Index count) { + _mm256_mask_storeu_epi16(to, static_cast<__mmask16>(segment_kmask(begin, count)), from); +} + +template <> +inline Packet8bf ploaduSegment<Packet8bf>(const bfloat16* from, Index begin, Index count) { + return _mm_maskz_loadu_epi16(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<bfloat16, Packet8bf>(bfloat16* to, const Packet8bf& from, Index begin, Index count) { + _mm_mask_storeu_epi16(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +#ifndef EIGEN_VECTORIZE_AVX512FP16 + +/*---------------- half ----------------*/ + +template <> +struct has_packet_segment<Packet16h> : std::true_type {}; + +template <> +struct has_packet_segment<Packet8h> : std::true_type {}; + +template <> +inline Packet16h ploaduSegment<Packet16h>(const Eigen::half* from, Index begin, Index count) { + return _mm256_maskz_loadu_epi16(static_cast<__mmask16>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<Eigen::half, Packet16h>(Eigen::half* to, const Packet16h& from, Index begin, Index count) { + _mm256_mask_storeu_epi16(to, static_cast<__mmask16>(segment_kmask(begin, count)), from); +} + +template <> +inline Packet8h ploaduSegment<Packet8h>(const Eigen::half* from, Index begin, Index count) { + return _mm_maskz_loadu_epi16(static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +template <> +inline void pstoreuSegment<Eigen::half, Packet8h>(Eigen::half* to, const Packet8h& from, Index begin, Index count) { + _mm_mask_storeu_epi16(to, static_cast<__mmask8>(segment_kmask(begin, count)), from); +} + +#endif // EIGEN_VECTORIZE_AVX512FP16 +#endif // __AVX512BW__ +#endif // EIGEN_VECTORIZE_AVX512VL + +/*---------------- end load/store segment support ----------------*/ + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h index 34b2939..d08941e 100644 --- a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h +++ b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h
@@ -1436,6 +1436,51 @@ from[2 * stride].x, from[1 * stride].x, from[0 * stride].x); } +/*---------------- load/store segment support ----------------*/ + +// There are no masked FP16 moves; the word-granular AVX-512BW ones move the same bits. + +template <> +struct has_packet_segment<Packet32h> : std::true_type {}; + +template <> +struct has_packet_segment<Packet16h> : std::true_type {}; + +template <> +struct has_packet_segment<Packet8h> : std::true_type {}; + +template <> +inline Packet32h ploaduSegment<Packet32h>(const Eigen::half* from, Index begin, Index count) { + return _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(static_cast<__mmask32>(segment_kmask(begin, count)), from)); +} + +template <> +inline void pstoreuSegment<Eigen::half, Packet32h>(Eigen::half* to, const Packet32h& from, Index begin, Index count) { + _mm512_mask_storeu_epi16(to, static_cast<__mmask32>(segment_kmask(begin, count)), _mm512_castph_si512(from)); +} + +template <> +inline Packet16h ploaduSegment<Packet16h>(const Eigen::half* from, Index begin, Index count) { + return _mm256_castsi256_ph(_mm256_maskz_loadu_epi16(static_cast<__mmask16>(segment_kmask(begin, count)), from)); +} + +template <> +inline void pstoreuSegment<Eigen::half, Packet16h>(Eigen::half* to, const Packet16h& from, Index begin, Index count) { + _mm256_mask_storeu_epi16(to, static_cast<__mmask16>(segment_kmask(begin, count)), _mm256_castph_si256(from)); +} + +template <> +inline Packet8h ploaduSegment<Packet8h>(const Eigen::half* from, Index begin, Index count) { + return _mm_castsi128_ph(_mm_maskz_loadu_epi16(static_cast<__mmask8>(segment_kmask(begin, count)), from)); +} + +template <> +inline void pstoreuSegment<Eigen::half, Packet8h>(Eigen::half* to, const Packet8h& from, Index begin, Index count) { + _mm_mask_storeu_epi16(to, static_cast<__mmask8>(segment_kmask(begin, count)), _mm_castph_si128(from)); +} + +/*---------------- end load/store segment support ----------------*/ + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/AVX512/Reductions.h b/Eigen/src/Core/arch/AVX512/Reductions.h index 2e93cce..7c81785 100644 --- a/Eigen/src/Core/arch/AVX512/Reductions.h +++ b/Eigen/src/Core/arch/AVX512/Reductions.h
@@ -232,7 +232,8 @@ // Count 16-bit floating-point lanes through integer bits so fast-math cannot treat NaN masks as zero. EIGEN_STRONG_INLINE Index predux_count_16bit(const __m256i& a) { #if defined(EIGEN_VECTORIZE_AVX512VL) && defined(__AVX512BW__) - const unsigned int nonzero_lanes = _cvtmask16_u32(_mm256_test_epi16_mask(a, _mm256_set1_epi16(0x7fff))); + // A cast rather than _cvtmask16_u32, whose nvc++ definition calls a __builtin_ia32_kmovw that nvc++ cannot link. + const unsigned int nonzero_lanes = static_cast<unsigned int>(_mm256_test_epi16_mask(a, _mm256_set1_epi16(0x7fff))); return Index(popcount(nonzero_lanes)); #else const __m256i magnitude = _mm256_and_si256(a, _mm256_set1_epi16(0x7fff));
diff --git a/Eigen/src/Core/arch/AltiVec/MatrixProduct.h b/Eigen/src/Core/arch/AltiVec/MatrixProduct.h index 697dbb4..38e5182 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixProduct.h +++ b/Eigen/src/Core/arch/AltiVec/MatrixProduct.h
@@ -1502,9 +1502,14 @@ } } -template <int N, typename Scalar, typename Packet, bool NegativeAccumulate> +template <int N, typename Scalar, typename Packet, bool NegativeAccumulate, Index remaining_rows = 0> EIGEN_ALWAYS_INLINE void pger(PacketBlock<Packet, N>* acc, const Scalar* lhs, const Packet* rhsV) { - Packet lhsV = pload<Packet>(lhs); + Packet lhsV; + EIGEN_IF_CONSTEXPR (remaining_rows > 0) { + lhsV = ploadu_partial<Packet>(lhs, remaining_rows); + } else { + lhsV = ploadLhs<Packet>(lhs); + } pger_common<Packet, NegativeAccumulate, N>(acc, lhsV, rhsV); } @@ -1529,15 +1534,26 @@ } } -template <int N, typename Scalar, typename Packet, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal> +template <int N, typename Scalar, typename Packet, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal, + Index remaining_rows = 0> EIGEN_ALWAYS_INLINE void pgerc(PacketBlock<Packet, N>* accReal, PacketBlock<Packet, N>* accImag, const Scalar* lhs_ptr, const Scalar* lhs_ptr_imag, const Packet* rhsV, const Packet* rhsVi) { - Packet lhsV = ploadLhs<Packet>(lhs_ptr); + Packet lhsV; + EIGEN_IF_CONSTEXPR (remaining_rows > 0) { + lhsV = ploadu_partial<Packet>(lhs_ptr, remaining_rows); + } else { + lhsV = ploadLhs<Packet>(lhs_ptr); + } Packet lhsVi; - EIGEN_IF_CONSTEXPR (!LhsIsReal) - lhsVi = ploadLhs<Packet>(lhs_ptr_imag); - else + EIGEN_IF_CONSTEXPR (!LhsIsReal) { + EIGEN_IF_CONSTEXPR (remaining_rows > 0) { + lhsVi = ploadu_partial<Packet>(lhs_ptr_imag, remaining_rows); + } else { + lhsVi = ploadLhs<Packet>(lhs_ptr_imag); + } + } else { EIGEN_UNUSED_VARIABLE(lhs_ptr_imag); + } pgerc_common<N, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi); } @@ -1625,17 +1641,19 @@ } } -#ifdef USE_PARTIAL_PACKETS +// When Complex && full, the first N packets are full loads and `elements` counts only the second half at row + accCols. template <typename DataMapper, typename Packet, const Index accCols, bool Complex, Index N, bool full> EIGEN_ALWAYS_INLINE void bload_partial(PacketBlock<Packet, N*(Complex ? 2 : 1)>& acc, const DataMapper& res, Index row, Index elements) { - for (Index M = 0; M < N; M++) { - acc.packet[M] = res.template loadPacketPartial<Packet>(row, M, elements); - } EIGEN_IF_CONSTEXPR (Complex && full) { for (Index M = 0; M < N; M++) { + acc.packet[M] = res.template loadPacket<Packet>(row, M); acc.packet[M + N] = res.template loadPacketPartial<Packet>(row + accCols, M, elements); } + } else { + for (Index M = 0; M < N; M++) { + acc.packet[M] = res.template loadPacketPartial<Packet>(row, M, elements); + } } } @@ -1645,7 +1663,6 @@ res.template storePacketPartial<Packet>(row, M, acc.packet[M], elements); } } -#endif #ifdef _ARCH_PWR10 #define USE_P10_AND_PVIPR2_0 (EIGEN_COMP_LLVM || (__GNUC__ >= 11)) @@ -1900,11 +1917,11 @@ MICRO_PREFETCHN1(ptr_imag, N); \ } -template <typename Scalar, typename Packet, const Index accRows, const Index remaining_rows> +template <typename Scalar, typename Packet, const Index accRows, const Index remaining_rows, const Index load_rows = 0> EIGEN_ALWAYS_INLINE void MICRO_EXTRA_ROW(const Scalar*& lhs_ptr, const Scalar*& rhs_ptr0, const Scalar*& rhs_ptr1, const Scalar*& rhs_ptr2, PacketBlock<Packet, accRows>& accZero) { MICRO_BROADCAST_EXTRA - pger<accRows, Scalar, Packet, false>(&accZero, lhs_ptr, rhsV); + pger<accRows, Scalar, Packet, false, load_rows>(&accZero, lhs_ptr, rhsV); lhs_ptr += remaining_rows; } @@ -1921,41 +1938,30 @@ MICRO_SRC2_PTR bsetzero<Packet, accRows>(accZero0); - Index remaining_depth = depth & -quad_traits<Scalar>::rows; + const Index peel_depth = depth - (accCols - remaining_rows); Index k = 0; - if (remaining_depth >= PEEL_ROW) { + if (peel_depth >= PEEL_ROW) { MICRO_ZERO_PEEL_ROW do { MICRO_PREFETCHN(accRows) EIGEN_POWER_PREFETCH(lhs_ptr); MICRO_WORK_PEEL_ROW - } while ((k += PEEL_ROW) + PEEL_ROW <= remaining_depth); + } while ((k += PEEL_ROW) + PEEL_ROW <= peel_depth); MICRO_ADD_PEEL_ROW } - for (; k < depth; k++) { + for (; k < peel_depth; k++) { MICRO_EXTRA_ROW<Scalar, Packet, accRows, remaining_rows>(lhs_ptr, rhs_ptr0, rhs_ptr1, rhs_ptr2, accZero0); } + for (; k < depth; k++) { + MICRO_EXTRA_ROW<Scalar, Packet, accRows, remaining_rows, remaining_rows>(lhs_ptr, rhs_ptr0, rhs_ptr1, rhs_ptr2, + accZero0); + } -#ifdef USE_PARTIAL_PACKETS EIGEN_UNUSED_VARIABLE(rows); EIGEN_UNUSED_VARIABLE(pMask); bload_partial<DataMapper, Packet, 0, false, accRows>(acc, res, row, remaining_rows); bscale<Packet, accRows>(acc, accZero0, pAlpha); bstore_partial<DataMapper, Packet, accRows>(acc, res, row, remaining_rows); -#else - bload<DataMapper, Packet, 0, ColMajor, false, accRows>(acc, res, row, 0); - if ((accRows == 1) || (rows >= accCols)) { - bscale<Packet, accRows, true>(acc, accZero0, pAlpha, pMask); - bstore<DataMapper, Packet, accRows>(acc, res, row); - } else { - bscale<Packet, accRows, false>(acc, accZero0, pAlpha, pMask); - for (Index j = 0; j < accRows; j++) { - for (Index i = 0; i < remaining_rows; i++) { - res(row + i, j) = acc.packet[j][i]; - } - } - } -#endif } #define MICRO_EXTRA(MICRO_EXTRA_UNROLL, value, is_col) \ @@ -2017,10 +2023,16 @@ MICRO_TYPE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE) \ MICRO_ADD_ROWS(size) +#define MICRO_UNROLL_TYPE_PARTIAL(MICRO_TYPE, size) \ + MICRO_TYPE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_PARTIAL_ONE) \ + MICRO_ADD_ROWS(size) + #define MICRO_ONE_PEEL4 MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_PEEL, PEEL) #define MICRO_ONE4 MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_ONE, 1) +#define MICRO_ONE4_PARTIAL MICRO_UNROLL_TYPE_PARTIAL(MICRO_UNROLL_TYPE_ONE, 1) + #define MICRO_DST_PTR_ONE(iter) \ EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ bsetzero<Packet, accRows>(accZero##iter); \ @@ -2034,7 +2046,6 @@ #define MICRO_PREFETCH MICRO_UNROLL(MICRO_PREFETCH_ONE) -#ifdef USE_PARTIAL_PACKETS #define MICRO_STORE_ONE(iter) \ EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ EIGEN_IF_CONSTEXPR (MICRO_NORMAL_PARTIAL(iter)) { \ @@ -2047,33 +2058,14 @@ bstore_partial<DataMapper, Packet, accRows>(acc, res, row + iter * accCols, accCols2); \ } \ } -#else -#define MICRO_STORE_ONE(iter) \ - EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ - bload<DataMapper, Packet, 0, ColMajor, false, accRows>(acc, res, row + iter * accCols, 0); \ - bscale<Packet, accRows, !(MICRO_NORMAL(iter))>(acc, accZero##iter, pAlpha, pMask); \ - bstore<DataMapper, Packet, accRows>(acc, res, row + iter * accCols); \ - } -#endif #define MICRO_STORE MICRO_UNROLL(MICRO_STORE_ONE) -#ifdef USE_PARTIAL_PACKETS template <int unroll_factor, typename Scalar, typename Packet, typename DataMapper, const Index accRows, const Index accCols, bool full> -#else -template <int unroll_factor, typename Scalar, typename Packet, typename DataMapper, const Index accRows, - const Index accCols, const Index accCols2> -#endif EIGEN_ALWAYS_INLINE void gemm_unrolled_iteration(const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base, Index depth, Index strideA, Index offsetA, Index strideB, Index& row, - const Packet& pAlpha, -#ifdef USE_PARTIAL_PACKETS - Index accCols2 -#else - const Packet& pMask -#endif -) { + const Packet& pAlpha, Index accCols2) { const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 = nullptr, *rhs_ptr2 = nullptr; const Scalar *lhs_ptr0 = nullptr, *lhs_ptr1 = nullptr, *lhs_ptr2 = nullptr, *lhs_ptr3 = nullptr, *lhs_ptr4 = nullptr, *lhs_ptr5 = nullptr, *lhs_ptr6 = nullptr, *lhs_ptr7 = nullptr; @@ -2084,31 +2076,30 @@ MICRO_SRC_PTR MICRO_DST_PTR + const Index peel_depth = full ? depth : (depth - (accCols - accCols2)); Index k = 0; - for (; k + PEEL <= depth; k += PEEL) { + for (; k + PEEL <= peel_depth; k += PEEL) { MICRO_PREFETCHN(accRows) MICRO_PREFETCH MICRO_ONE_PEEL4 } - for (; k < depth; k++) { + for (; k < peel_depth; k++) { MICRO_ONE4 } + EIGEN_IF_CONSTEXPR (!full) { + for (; k < depth; k++) { + MICRO_ONE4_PARTIAL + } + } MICRO_STORE MICRO_UPDATE } -#ifdef USE_PARTIAL_PACKETS #define MICRO_UNROLL_ITER2(N, M) \ gemm_unrolled_iteration<N + ((M) ? 1 : 0), Scalar, Packet, DataMapper, accRows, accCols, !M>( \ res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlpha, M ? remaining_rows : accCols); \ if (M) return; -#else -#define MICRO_UNROLL_ITER2(N, M) \ - gemm_unrolled_iteration<N + ((M) ? 1 : 0), Scalar, Packet, DataMapper, accRows, accCols, M ? M : accCols>( \ - res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlpha, pMask); \ - if (M) return; -#endif template <typename Scalar, typename Packet, typename DataMapper, const Index accRows, const Index accCols> EIGEN_ALWAYS_INLINE void gemm_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index depth, @@ -2308,7 +2299,7 @@ MICRO_COMPLEX_ADD_PEEL(2, 0) MICRO_COMPLEX_ADD_PEEL(3, 1) MICRO_COMPLEX_ADD_PEEL(1, 0) template <typename Scalar, typename Packet, const Index accRows, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, - bool RhsIsReal, const Index remaining_rows> + bool RhsIsReal, const Index remaining_rows, const Index load_rows = 0> EIGEN_ALWAYS_INLINE void MICRO_COMPLEX_EXTRA_ROW(const Scalar*& lhs_ptr_real, const Scalar*& lhs_ptr_imag, const Scalar*& rhs_ptr_real0, const Scalar*& rhs_ptr_real1, const Scalar*& rhs_ptr_real2, const Scalar*& rhs_ptr_imag0, @@ -2316,8 +2307,8 @@ PacketBlock<Packet, accRows>& accReal, PacketBlock<Packet, accRows>& accImag) { MICRO_COMPLEX_BROADCAST_EXTRA - pgerc<accRows, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal, &accImag, lhs_ptr_real, - lhs_ptr_imag, rhsV, rhsVi); + pgerc<accRows, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, load_rows>( + &accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi); MICRO_COMPLEX_ADD_COLS(1) } @@ -2347,9 +2338,9 @@ bsetzero<Packet, accRows>(accReal0); bsetzero<Packet, accRows>(accImag0); - Index remaining_depth = depth & -quad_traits<Scalar>::rows; + const Index peel_depth = depth - (accCols - remaining_rows); Index k = 0; - if (remaining_depth >= PEEL_COMPLEX_ROW) { + if (peel_depth >= PEEL_COMPLEX_ROW) { MICRO_COMPLEX_ZERO_PEEL_ROW do { MICRO_COMPLEX_PREFETCHN(accRows) @@ -2358,39 +2349,40 @@ EIGEN_POWER_PREFETCH(lhs_ptr_imag); } MICRO_COMPLEX_WORK_PEEL_ROW - } while ((k += PEEL_COMPLEX_ROW) + PEEL_COMPLEX_ROW <= remaining_depth); + } while ((k += PEEL_COMPLEX_ROW) + PEEL_COMPLEX_ROW <= peel_depth); MICRO_COMPLEX_ADD_PEEL_ROW } - for (; k < depth; k++) { + for (; k < peel_depth; k++) { MICRO_COMPLEX_EXTRA_ROW<Scalar, Packet, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, remaining_rows>( lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real0, rhs_ptr_real1, rhs_ptr_real2, rhs_ptr_imag0, rhs_ptr_imag1, rhs_ptr_imag2, accReal0, accImag0); } + for (; k < depth; k++) { + MICRO_COMPLEX_EXTRA_ROW<Scalar, Packet, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, remaining_rows, + remaining_rows>(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real0, rhs_ptr_real1, rhs_ptr_real2, + rhs_ptr_imag0, rhs_ptr_imag1, rhs_ptr_imag2, accReal0, accImag0); + } + EIGEN_UNUSED_VARIABLE(rows); constexpr bool full = (remaining_rows > accColsC); - bload<DataMapper, Packetc, accColsC, ColMajor, true, accRows, full>(tRes, res, row, 0); - if ((accRows == 1) || (rows >= accCols)) { - bscalec<Packet, accRows, true>(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); - bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1); - bstore<DataMapper, Packetc, accRows>(acc0, res, row + 0); - EIGEN_IF_CONSTEXPR (full) { - bstore<DataMapper, Packetc, accRows>(acc1, res, row + accColsC); - } + constexpr bool odd = (sizeof(Scalar) == sizeof(float)) && (remaining_rows & 1); + EIGEN_IF_CONSTEXPR (odd) { + bload_partial<DataMapper, Packetc, accColsC, true, accRows, full>(tRes, res, row, 1); } else { - bscalec<Packet, accRows, false>(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); - bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1); - - if ((sizeof(Scalar) == sizeof(float)) && (remaining_rows == 1)) { - for (Index j = 0; j < accRows; j++) { - res(row + 0, j) = pfirst<Packetc>(acc0.packet[j]); - } + bload<DataMapper, Packetc, accColsC, ColMajor, true, accRows, full>(tRes, res, row, 0); + } + bscalec<Packet, accRows, true>(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask); + bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1); + EIGEN_IF_CONSTEXPR (odd && !full) { + bstore_partial<DataMapper, Packetc, accRows>(acc0, res, row + 0, 1); + } else { + bstore<DataMapper, Packetc, accRows>(acc0, res, row + 0); + } + EIGEN_IF_CONSTEXPR (full) { + EIGEN_IF_CONSTEXPR (odd) { + bstore_partial<DataMapper, Packetc, accRows>(acc1, res, row + accColsC, 1); } else { - bstore<DataMapper, Packetc, accRows>(acc0, res, row + 0); - EIGEN_IF_CONSTEXPR (full) { - for (Index j = 0; j < accRows; j++) { - res(row + accColsC, j) = pfirst<Packetc>(acc1.packet[j]); - } - } + bstore<DataMapper, Packetc, accRows>(acc1, res, row + accColsC); } } } @@ -2443,10 +2435,16 @@ MICRO_COMPLEX_TYPE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE) \ MICRO_COMPLEX_ADD_ROWS(size, false) +#define MICRO_COMPLEX_UNROLL_TYPE_PARTIAL(MICRO_COMPLEX_TYPE, size) \ + MICRO_COMPLEX_TYPE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_PARTIAL_ONE) \ + MICRO_COMPLEX_ADD_ROWS(size, false) + #define MICRO_COMPLEX_ONE_PEEL4 MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_PEEL, PEEL_COMPLEX) #define MICRO_COMPLEX_ONE4 MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1) +#define MICRO_COMPLEX_ONE4_PARTIAL MICRO_COMPLEX_UNROLL_TYPE_PARTIAL(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1) + #define MICRO_COMPLEX_DST_PTR_ONE(iter) \ EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ bsetzero<Packet, accRows>(accReal##iter); \ @@ -2465,13 +2463,26 @@ #define MICRO_COMPLEX_STORE_ONE(iter) \ EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \ constexpr bool full = ((MICRO_NORMAL(iter)) || (accCols2 > accColsC)); \ - bload<DataMapper, Packetc, accColsC, ColMajor, true, accRows, full>(tRes, res, row + iter * accCols, 0); \ + constexpr bool odd = !(MICRO_NORMAL(iter)) && (sizeof(Scalar) == sizeof(float)) && (accCols2 & 1); \ + EIGEN_IF_CONSTEXPR (odd) { \ + bload_partial<DataMapper, Packetc, accColsC, true, accRows, full>(tRes, res, row + iter * accCols, 1); \ + } else { \ + bload<DataMapper, Packetc, accColsC, ColMajor, true, accRows, full>(tRes, res, row + iter * accCols, 0); \ + } \ bscalec<Packet, accRows, !(MICRO_NORMAL(iter))>(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, \ taccImag, pMask); \ bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1); \ - bstore<DataMapper, Packetc, accRows>(acc0, res, row + iter * accCols + 0); \ + EIGEN_IF_CONSTEXPR (odd && !full) { \ + bstore_partial<DataMapper, Packetc, accRows>(acc0, res, row + iter * accCols + 0, 1); \ + } else { \ + bstore<DataMapper, Packetc, accRows>(acc0, res, row + iter * accCols + 0); \ + } \ EIGEN_IF_CONSTEXPR (full) { \ - bstore<DataMapper, Packetc, accRows>(acc1, res, row + iter * accCols + accColsC); \ + EIGEN_IF_CONSTEXPR (odd) { \ + bstore_partial<DataMapper, Packetc, accRows>(acc1, res, row + iter * accCols + accColsC, 1); \ + } else { \ + bstore<DataMapper, Packetc, accRows>(acc1, res, row + iter * accCols + accColsC); \ + } \ } \ } @@ -2501,15 +2512,21 @@ MICRO_COMPLEX_SRC_PTR MICRO_COMPLEX_DST_PTR + const Index peel_depth = depth - (accCols - accCols2); Index k = 0; - for (; k + PEEL_COMPLEX <= depth; k += PEEL_COMPLEX) { + for (; k + PEEL_COMPLEX <= peel_depth; k += PEEL_COMPLEX) { MICRO_COMPLEX_PREFETCHN(accRows) MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_ONE_PEEL4 } - for (; k < depth; k++) { + for (; k < peel_depth; k++) { MICRO_COMPLEX_ONE4 } + EIGEN_IF_CONSTEXPR (accCols != accCols2) { + for (; k < depth; k++) { + MICRO_COMPLEX_ONE4_PARTIAL + } + } MICRO_COMPLEX_STORE MICRO_COMPLEX_UPDATE
diff --git a/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h b/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h index e7b69bf..63f5eb8 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h +++ b/Eigen/src/Core/arch/AltiVec/MatrixProductCommon.h
@@ -11,10 +11,6 @@ #define EIGEN_POWER_PREFETCH(p) #endif -#if defined(_ARCH_PWR9) || defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH) -#define USE_PARTIAL_PACKETS -#endif - // IWYU pragma: private #include "../../InternalHeaderCheck.h" @@ -86,14 +82,12 @@ template <typename DataMapper, typename Packet, int N> EIGEN_ALWAYS_INLINE void bstore(PacketBlock<Packet, N>& acc, const DataMapper& res, Index row); -#ifdef USE_PARTIAL_PACKETS template <typename DataMapper, typename Packet, const Index accCols, bool Complex, Index N, bool full = true> EIGEN_ALWAYS_INLINE void bload_partial(PacketBlock<Packet, N*(Complex ? 2 : 1)>& acc, const DataMapper& res, Index row, Index elements); template <typename DataMapper, typename Packet, Index N> EIGEN_ALWAYS_INLINE void bstore_partial(PacketBlock<Packet, N>& acc, const DataMapper& res, Index row, Index elements); -#endif template <typename Packet, int N> EIGEN_ALWAYS_INLINE void bscale(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ, const Packet& pAlpha); @@ -132,7 +126,6 @@ break; \ } -#ifdef USE_PARTIAL_PACKETS #define MICRO_UNROLL_ITER(func, N) \ if (remaining_rows) { \ func(N, true); \ @@ -141,9 +134,6 @@ } #define MICRO_NORMAL_PARTIAL(iter) full || (unroll_factor != (iter + 1)) -#else -#define MICRO_UNROLL_ITER(func, N) MICRO_UNROLL_ITER1(func, N) -#endif #define MICRO_COMPLEX_UNROLL_ITER(func, N) MICRO_UNROLL_ITER1(func, N) @@ -167,6 +157,33 @@ } \ MICRO_LOAD1(lhs_ptr_real, iter) +#define MICRO_LOAD1_PARTIAL(lhs_ptr, iter) \ + if (unroll_factor > iter) { \ + if (MICRO_NORMAL(iter)) { \ + lhsV##iter = ploadLhs<Packet>(lhs_ptr##iter); \ + lhs_ptr##iter += accCols; \ + } else { \ + lhsV##iter = ploadu_partial<Packet>(lhs_ptr##iter, accCols2); \ + lhs_ptr##iter += accCols2; \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(lhsV##iter); \ + } + +#define MICRO_LOAD_PARTIAL_ONE(iter) MICRO_LOAD1_PARTIAL(lhs_ptr, iter) + +#define MICRO_COMPLEX_LOAD_PARTIAL_ONE(iter) \ + if (!LhsIsReal && (unroll_factor > iter)) { \ + if (MICRO_NORMAL(iter)) { \ + lhsVi##iter = ploadLhs<Packet>(lhs_ptr_real##iter + imag_delta); \ + } else { \ + lhsVi##iter = ploadu_partial<Packet>(lhs_ptr_real##iter + imag_delta2, accCols2); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ + } \ + MICRO_LOAD1_PARTIAL(lhs_ptr_real, iter) + #define MICRO_SRC_PTR1(lhs_ptr, advRows, iter) \ if (unroll_factor > iter) { \ lhs_ptr##iter = lhs_base + (row + (iter * accCols)) * strideA * advRows - \ @@ -188,15 +205,8 @@ #define MICRO_COMPLEX_PREFETCH_ONE(iter) MICRO_PREFETCH1(lhs_ptr_real, iter) -#ifdef USE_PARTIAL_PACKETS -#define MICRO_UPDATE_MASK -#else -#define MICRO_UPDATE_MASK EIGEN_UNUSED_VARIABLE(pMask); -#endif - #define MICRO_UPDATE \ if (accCols == accCols2) { \ - MICRO_UPDATE_MASK \ EIGEN_UNUSED_VARIABLE(offsetA); \ row += unroll_factor * accCols; \ }
diff --git a/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h b/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h index 37d5b46..f5c219b 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h +++ b/Eigen/src/Core/arch/AltiVec/MatrixProductMMA.h
@@ -64,12 +64,17 @@ const Packet& alphaImag, const Packet& pMask, __vector_quad* accReal, __vector_quad* accImag) { constexpr bool full = (accCols2 > accColsC); + constexpr bool odd = (accCols != accCols2) && (sizeof(__UNPACK_TYPE__(Packet)) == sizeof(float)) && (accCols2 & 1); PacketBlock<Packet, 4> resultReal, resultImag; __builtin_mma_disassemble_acc(&resultReal.packet, accReal); __builtin_mma_disassemble_acc(&resultImag.packet, accImag); PacketBlock<Packetc, 8> tRes; - bload<DataMapper, Packetc, accColsC, ColMajor, true, 4, full>(tRes, data, i, 0); + EIGEN_IF_CONSTEXPR (odd) { + bload_partial<DataMapper, Packetc, accColsC, true, 4, full>(tRes, data, i, 1); + } else { + bload<DataMapper, Packetc, accColsC, ColMajor, true, 4, full>(tRes, data, i, 0); + } PacketBlock<Packet, 4> taccReal, taccImag; bscalec<Packet, 4, (accCols != accCols2)>(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag, pMask); @@ -77,9 +82,17 @@ PacketBlock<Packetc, 4> acc1, acc2; bcouple<Packet, Packetc, 4, full>(taccReal, taccImag, tRes, acc1, acc2); - bstore<DataMapper, Packetc, 4>(acc1, data, i); - if (full) { - bstore<DataMapper, Packetc, 4>(acc2, data, i + accColsC); + EIGEN_IF_CONSTEXPR (odd && !full) { + bstore_partial<DataMapper, Packetc, 4>(acc1, data, i, 1); + } else { + bstore<DataMapper, Packetc, 4>(acc1, data, i); + } + EIGEN_IF_CONSTEXPR (full) { + EIGEN_IF_CONSTEXPR (odd) { + bstore_partial<DataMapper, Packetc, 4>(acc2, data, i + accColsC, 1); + } else { + bstore<DataMapper, Packetc, 4>(acc2, data, i + accColsC); + } } } @@ -280,6 +293,10 @@ MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_ONE, RhsPacket) \ MICRO_MMA_UPDATE_RHS(size) +#define MICRO_MMA_UNROLL_TYPE_PARTIAL(MICRO_MMA_TYPE, size) \ + MICRO_MMA_TYPE(MICRO_MMA_WORK_ONE, MICRO_LOAD_PARTIAL_ONE, RhsPacket) \ + MICRO_MMA_UPDATE_RHS(size) + #ifndef VECTOR_PAIR_LOADS_LHS #define MICRO_MMA_ONE_PEEL MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_PEEL, PEEL_MMA) #else @@ -292,6 +309,8 @@ #define MICRO_MMA_ONE MICRO_MMA_UNROLL_TYPE(MICRO_MMA_UNROLL_TYPE_ONE, 1) +#define MICRO_MMA_ONE_PARTIAL MICRO_MMA_UNROLL_TYPE_PARTIAL(MICRO_MMA_UNROLL_TYPE_ONE, 1) + #define MICRO_MMA_DST_PTR_ONE(iter) \ if (unroll_factor * accItr > iter) { \ bsetzeroMMA(&accZero##iter); \ @@ -361,15 +380,21 @@ MICRO_MMA_SRC_PTR MICRO_MMA_DST_PTR - Index k = 0, depth2 = depth - PEEL_MMA; + const Index peel_depth = full ? depth : (depth - (accCols - accCols2)); + Index k = 0, depth2 = peel_depth - PEEL_MMA; for (; k <= depth2; k += PEEL_MMA) { EIGEN_POWER_PREFETCH(rhs_ptr); MICRO_MMA_PREFETCH MICRO_MMA_ONE_PEEL } - for (; k < depth; k++) { + for (; k < peel_depth; k++) { MICRO_MMA_ONE } + EIGEN_IF_CONSTEXPR (!full) { + for (; k < depth; k++) { + MICRO_MMA_ONE_PARTIAL + } + } MICRO_MMA_STORE MICRO_UPDATE @@ -641,6 +666,10 @@ MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_ONE, RhsPacket) \ MICRO_COMPLEX_MMA_UPDATE_RHS(size); +#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PARTIAL(MICRO_COMPLEX_MMA_TYPE, size) \ + MICRO_COMPLEX_MMA_TYPE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_LOAD_PARTIAL_ONE, RhsPacket) \ + MICRO_COMPLEX_MMA_UPDATE_RHS(size); + #ifndef VECTOR_PAIR_LOADS_LHS #define MICRO_COMPLEX_MMA_ONE_PEEL MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL, PEEL_COMPLEX_MMA) #else @@ -654,6 +683,8 @@ #define MICRO_COMPLEX_MMA_ONE MICRO_COMPLEX_MMA_UNROLL_TYPE(MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE, 1) +#define MICRO_COMPLEX_MMA_ONE_PARTIAL MICRO_COMPLEX_MMA_UNROLL_TYPE_PARTIAL(MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE, 1) + #define MICRO_COMPLEX_MMA_DST_PTR_ONE(iter) \ if (unroll_factor * accItr > iter) { \ bsetzeroMMA(&accReal##iter); \ @@ -753,7 +784,8 @@ MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_DST_PTR - Index k = 0, depth2 = depth - PEEL_COMPLEX_MMA; + const Index peel_depth = depth - (accCols - accCols2); + Index k = 0, depth2 = peel_depth - PEEL_COMPLEX_MMA; for (; k <= depth2; k += PEEL_COMPLEX_MMA) { EIGEN_POWER_PREFETCH(rhs_ptr_real); if (!RhsIsReal) { @@ -762,9 +794,14 @@ MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_ONE_PEEL } - for (; k < depth; k++) { + for (; k < peel_depth; k++) { MICRO_COMPLEX_MMA_ONE } + EIGEN_IF_CONSTEXPR (accCols != accCols2) { + for (; k < depth; k++) { + MICRO_COMPLEX_MMA_ONE_PARTIAL + } + } MICRO_COMPLEX_MMA_STORE MICRO_COMPLEX_UPDATE
diff --git a/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc b/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc index acbd03d..f9d4b32 100644 --- a/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc +++ b/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc
@@ -1020,7 +1020,7 @@ /** \internal packet conjugate with real & imaginary operation inverted */ EIGEN_ALWAYS_INLINE Packet2cf pconjinv(const Packet2cf& a) { -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR return Packet2cf(Packet4f(vec_neg(Packet2d(a.v)))); #else return Packet2cf(pxor(a.v, reinterpret_cast<Packet4f>(p16uc_COMPLEX32_CONJ_XOR2))); @@ -1071,7 +1071,7 @@ /** \internal packet negate */ EIGEN_ALWAYS_INLINE Packet2cf pnegate2(const Packet2cf& a) { -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR return Packet2cf(vec_neg(a.v)); #else return Packet2cf(pxor(a.v, reinterpret_cast<Packet4f>(p16uc_COMPLEX32_NEGATE))); @@ -1079,7 +1079,7 @@ } EIGEN_ALWAYS_INLINE Packet1cd pnegate2(const Packet1cd& a) { -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR return Packet1cd(vec_neg(a.v)); #else return Packet1cd(pxor(a.v, reinterpret_cast<Packet2d>(p16uc_COMPLEX64_NEGATE))); @@ -1153,7 +1153,7 @@ #endif } -#ifndef __POWER8_VECTOR__ +#ifndef EIGEN_VECTORIZE_POWER8_VECTOR const Packet16uc p16uc_MERGEE = {0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13, 0x08, 0x09, 0x0A, 0x0B, 0x18, 0x19, 0x1A, 0x1B}; @@ -1165,7 +1165,7 @@ template <typename RhsScalar> EIGEN_ALWAYS_INLINE void pload_realimag_row(RhsScalar* src, Packet4f& r, Packet4f& i) { Packet4f t = ploadu<Packet4f>(reinterpret_cast<float*>(src)); -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR r = vec_mergee(t, t); i = vec_mergeo(t, t); #else
diff --git a/Eigen/src/Core/arch/AltiVec/PacketMath.h b/Eigen/src/Core/arch/AltiVec/PacketMath.h index fbf728c..3a645a1 100644 --- a/Eigen/src/Core/arch/AltiVec/PacketMath.h +++ b/Eigen/src/Core/arch/AltiVec/PacketMath.h
@@ -1069,7 +1069,7 @@ template <> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR return vec_neg(a); #else return vec_xor(a, p4f_MZERO); @@ -1077,7 +1077,7 @@ } template <> EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) { -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR return vec_neg(a); #else return reinterpret_cast<Packet16c>(p4i_ZERO) - a; @@ -1085,7 +1085,7 @@ } template <> EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) { -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR return vec_neg(a); #else return reinterpret_cast<Packet8s>(p4i_ZERO) - a; @@ -1093,7 +1093,7 @@ } template <> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR return vec_neg(a); #else return p4i_ZERO - a; @@ -2437,6 +2437,18 @@ BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_eq<Packet4f>, a, b); } +// Compare encoded lanes: widening bf16 subnormals to float loses them under DAZ/FZ. +template <> +EIGEN_STRONG_INLINE Packet8bf psign<Packet8bf>(const Packet8bf& a) { + const Packet8us magnitude = vec_and(a.m_val, vec_splats(static_cast<unsigned short int>(0x7fff))); + const Packet8bi is_zero = vec_cmpeq(magnitude, vec_splats(static_cast<unsigned short int>(0))); + const Packet8us keep = + vec_or(reinterpret_cast<Packet8us>(vec_cmpgt(magnitude, vec_splats(static_cast<unsigned short int>(0x7f80)))), + vec_splats(static_cast<unsigned short int>(0x8000))); + const Packet8us value = vec_sel(vec_splats(static_cast<unsigned short int>(0x3f80)), a.m_val, keep); + return Packet8bf(vec_andc(value, reinterpret_cast<Packet8us>(is_zero))); +} + template <> EIGEN_STRONG_INLINE bfloat16 pfirst(const Packet8bf& a) { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16((pfirst<Packet8us>(a))); @@ -3298,7 +3310,7 @@ template <> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR return vec_neg(a); #else return vec_xor(a, p2d_MZERO); @@ -3361,7 +3373,7 @@ return reinterpret_cast<Packet2d>(vec_cmpeq(a, b)); } template <> -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR EIGEN_STRONG_INLINE Packet2l pcmp_eq(const Packet2l& a, const Packet2l& b) { return reinterpret_cast<Packet2l>(vec_cmpeq(a, b)); } @@ -3372,6 +3384,19 @@ return reinterpret_cast<Packet2l>(pand(halves, flipped)); } #endif +// Not the generic a < b ? ptrue(a) : pzero(a): under Clang's default -faltivec-src-compat=mixed, +// a < b on vector long long is a scalar all-lanes predicate, not a lane mask. +template <> +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR +EIGEN_STRONG_INLINE Packet2l pcmp_lt(const Packet2l& a, const Packet2l& b) { + return reinterpret_cast<Packet2l>(vec_cmplt(a, b)); +} +#else +EIGEN_STRONG_INLINE Packet2l pcmp_lt(const Packet2l& a, const Packet2l& b) { + const Packet2l ret = {a[0] < b[0] ? -1 : 0, a[1] < b[1] ? -1 : 0}; + return ret; +} +#endif template <> EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(const Packet2d& a, const Packet2d& b) { Packet2d c = reinterpret_cast<Packet2d>(vec_cmpge(a, b)); @@ -3481,7 +3506,7 @@ EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vec_abs(a); } -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR template <> EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) { return (Packet2d)vec_sra((Packet2l)a, vec_splats((unsigned long long)(63))); @@ -3512,7 +3537,7 @@ // Things are more complicated for POWER7. There is actually a // vec_xxsxdi intrinsic but it is not supported by some gcc versions. // So we need to shift by N % 32 and rearrange bytes. -#ifdef __POWER8_VECTOR__ +#ifdef EIGEN_VECTORIZE_POWER8_VECTOR template <int N> EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) { @@ -3619,19 +3644,17 @@ template <> EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) { - // Clamp exponent to [-2099, 2099] + // The single-rounding split of pldexp_generic on int64 exponents. const Packet2d max_exponent = pset1<Packet2d>(2099.0); + const Packet2d last_max = pset1<Packet2d>(1022.0); const Packet2l e = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); - - // Split 2^e into four factors and multiply: + const Packet2l b = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(last_max)), last_max)); const Packet2l bias = {1023, 1023}; - Packet2l b = plogical_shift_right<2>(e); // floor(e/4) - Packet2d c = reinterpret_cast<Packet2d>(plogical_shift_left<52>(b + bias)); - Packet2d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b) - b = psub(psub(psub(e, b), b), b); // e - 3b - c = reinterpret_cast<Packet2d>(plogical_shift_left<52>(b + bias)); // 2^(e - 3b) - out = pmul(out, c); // a * 2^e - return out; + const Packet2l even = {-2, -2}; + const Packet2l t = psub(e, b) & even; + const Packet2d c1 = reinterpret_cast<Packet2d>(plogical_shift_left<51>(t + bias + bias)); // 2^(t/2) + const Packet2d c2 = reinterpret_cast<Packet2d>(plogical_shift_left<52>(psub(e, t) + bias)); // 2^(e - t) + return pldexp_apply_factors(a, c1, c2); // a * 2^e } // Extract exponent without existence of Packet2l. @@ -3655,8 +3678,9 @@ template <> EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) { - const Packet2ul zero = {0, 0}; - return vec_any_ne(reinterpret_cast<Packet2ul>(a), zero); + // A 64-bit lane is nonzero iff one of its 32-bit halves is; the doubleword compare needs POWER8. + const Packet4ui zero = {0, 0, 0, 0}; + return vec_any_ne(reinterpret_cast<Packet4ui>(a), zero); } // Other reduction functions:
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathDoubleWord.h b/Eigen/src/Core/arch/Default/GenericPacketMathDoubleWord.h index c33616d..9e3acc2 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathDoubleWord.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathDoubleWord.h
@@ -138,27 +138,32 @@ #endif // EIGEN_VECTORIZE_FMA -// This function implements Dekker's algorithm for the addition -// of two double word numbers represented by {x_hi, x_lo} and {y_hi, y_lo}. -// It returns the result as a pair {s_hi, s_lo} such that -// x_hi + x_lo + y_hi + y_lo = s_hi + s_lo holds exactly. -// This is Algorithm 5 from Jean-Michel Muller, "Elementary Functions", -// 3rd edition, Birkh\"auser, 2016. +// This function implements the addition of two double word numbers +// represented by {x_hi, x_lo} and {y_hi, y_lo}, with operands of either sign +// and magnitude: Knuth's two-sum of the high parts is error-free, the low +// parts are added to its error term, and the result is renormalized. +// This is Algorithm 5 (SloppyDWPlusDW) from Joldes, Muller, & Popescu (2017), +// "Tight and rigorous error bounds for basic building blocks of double-word +// arithmetic", https://hal.science/hal-01351529: the error is a few u^2 +// relative to |x| + |y|, not to the sum, which may cancel. template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void twosum(const Packet& x_hi, const Packet& x_lo, const Packet& y_hi, const Packet& y_lo, Packet& s_hi, Packet& s_lo) { - const Packet x_greater_mask = pcmp_lt(pabs(y_hi), pabs(x_hi)); - Packet r_hi_1, r_lo_1; - fast_twosum(x_hi, y_hi, r_hi_1, r_lo_1); - Packet r_hi_2, r_lo_2; - fast_twosum(y_hi, x_hi, r_hi_2, r_lo_2); - const Packet r_hi = pselect(x_greater_mask, r_hi_1, r_hi_2); + Packet s = padd(x_hi, y_hi); + Packet y_part = psub(s, x_hi); + Packet err = padd(psub(x_hi, psub(s, y_part)), psub(y_hi, y_part)); + fast_twosum(s, padd(err, padd(x_lo, y_lo)), s_hi, s_lo); +} - const Packet s1 = padd(padd(y_lo, r_lo_1), x_lo); - const Packet s2 = padd(padd(x_lo, r_lo_2), y_lo); - const Packet s = pselect(x_greater_mask, s1, s2); - - fast_twosum(r_hi, s, s_hi, s_lo); +// The difference {x_hi, x_lo} - {y_hi, y_lo} of two double word numbers, +// the same algorithm as twosum with the sign of y folded into the operations. +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void twodiff(const Packet& x_hi, const Packet& x_lo, const Packet& y_hi, + const Packet& y_lo, Packet& s_hi, Packet& s_lo) { + Packet s = psub(x_hi, y_hi); + Packet y_part = psub(x_hi, s); + Packet err = psub(psub(x_hi, padd(s, y_part)), psub(y_hi, y_part)); + fast_twosum(s, padd(err, psub(x_lo, y_lo)), s_hi, s_lo); } // This is a version of twosum for double word numbers, @@ -225,6 +230,22 @@ fast_twosum(p_hi_hi, p_hi_lo, p_lo_hi, p_lo_lo, p_hi, p_lo); } +// This function implements the multiplication of two double word numbers +// {x_hi, x_lo} and {y_hi, y_lo} at a lower cost than twoprod above: the +// cross terms are accumulated onto the two-product residual and the x_lo*y_lo +// term is dropped. For normalized inputs the relative error is < 7*u^2. +// This is Algorithm 10 (DWTimesDW1) from Joldes, Muller, & Popescu (2017), +// "Tight and rigorous error bounds for basic building blocks of double-word +// arithmetic", https://hal.science/hal-01351529. +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void fast_twoprod(const Packet& x_hi, const Packet& x_lo, const Packet& y_hi, + const Packet& y_lo, Packet& p_hi, Packet& p_lo) { + Packet c_hi, c_lo; + twoprod(x_hi, y_hi, c_hi, c_lo); + c_lo = pmadd(x_hi, y_lo, pmadd(x_lo, y_hi, c_lo)); + fast_twosum(c_hi, c_lo, p_hi, p_lo); +} + // This function implements the division of double word {x_hi, x_lo} // by float y. This is Algorithm 15 from "Tight and rigorous error bounds // for basic building blocks of double-word arithmetic", Joldes, Muller, & Popescu,
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathFrexpLdexp.h b/Eigen/src/Core/arch/Default/GenericPacketMathFrexpLdexp.h index 5df1be5..4516b05 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathFrexpLdexp.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathFrexpLdexp.h
@@ -88,6 +88,17 @@ return m; } +// ((a * c1) * c1) * c2, with each partial product kept: -ffast-math would otherwise reassociate the factors, whose +// products overflow or underflow where a * 2^e does not (see pldexp_generic). +template <typename Packet> +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pldexp_apply_factors(const Packet& a, const Packet& c1, const Packet& c2) { + Packet out = pmul(a, c1); + EIGEN_OPTIMIZATION_BARRIER(out) + out = pmul(out, c1); + EIGEN_OPTIMIZATION_BARRIER(out) + return pmul(out, c2); +} + // Safely applies ldexp, correctly handles overflows, underflows and denormals. // Assumes IEEE floating point format. template <typename Packet> @@ -104,37 +115,46 @@ // denormal. // // Unfortunately, 2^(278) cannot be represented using either one or two - // finite normal floats, so we must split the scale factor into at least - // three parts. It turns out to be faster to split 'exponent' into four - // factors, since [exponent>>2] is much faster to compute than [exponent/3]. + // finite normal floats, so we must split the scale factor into three parts. + // A product rounded in two steps can be off by one unit when an + // intermediate is subnormal, so only the last multiplication may round: // - // Set e = min(max(exponent, -278), 278); - // b = floor(e/4); - // c1 = 2^b - // c2 = 2^(e - 3b) - // out = (((a * c1) * c1) * c1) * c2 (= a * 2^e) + // Set e = min(max(exponent, -278), 278); + // b = min(max(exponent, -126), 126); + // t = (e - b) & ~1 (even, |t| <= 152) + // c1 = 2^(t/2) + // c2 = 2^(e - t) (|e - t| <= 127) + // out = ((a * c1) * c1) * c2 (= a * 2^e) + // + // For |e| <= 126, t = 0 and the result is the one product a * 2^e. For + // e > 126 the first two products are exact until they overflow, and then + // the result overflows. For e < -126, t >= e + 125, so an inexact partial + // product, below 2^-126, means the result is below 2^-251 and rounds to + // zero either way. // // Every partial product must contain 'a'. Reassociating scale factors can // overflow (for example c1*c1 at e=256 for float), making pldexp(0, 256) - // NaN and finite denormal results infinite. Apply c1 before c2 because c2 - // may exceed one for negative exponents (e.g. c2=4 for e=-1), overflowing - // values near max even when the final result is finite. + // NaN and finite denormal results infinite. using PacketI = typename unpacket_traits<Packet>::integer_packet; using Scalar = typename unpacket_traits<Packet>::type; using ScalarI = typename unpacket_traits<PacketI>::type; static constexpr int TotalBits = sizeof(Scalar) * CHAR_BIT, MantissaBits = numext::numeric_limits<Scalar>::digits - 1, ExponentBits = TotalBits - MantissaBits - 1; + constexpr ScalarI bias_value = (ScalarI(1) << (ExponentBits - 1)) - ScalarI(1); // 127 constexpr ScalarI max_exp_value = (ScalarI(1) << ExponentBits) + ScalarI(MantissaBits - 1); // 278 + constexpr ScalarI last_max_value = bias_value - ScalarI(1); // 126 const Packet max_exponent = pset1<Packet>(Scalar(max_exp_value)); const Packet neg_max_exponent = pset1<Packet>(Scalar(-max_exp_value)); - const PacketI bias = pset1<PacketI>((ScalarI(1) << (ExponentBits - 1)) - ScalarI(1)); // 127 + const Packet last_max = pset1<Packet>(Scalar(last_max_value)); + const Packet neg_last_max = pset1<Packet>(Scalar(-last_max_value)); + const PacketI bias = pset1<PacketI>(bias_value); const PacketI e = pcast<Packet, PacketI>(pmin(pmax(exponent, neg_max_exponent), max_exponent)); - const PacketI b = parithmetic_shift_right<2>(e); // floor(e/4); - const PacketI b_remainder = pnmadd(pset1<PacketI>(3), b, e); // e - 3b - const Packet c1 = preinterpret<Packet>(plogical_shift_left<MantissaBits>(padd(b, bias))); // 2^b - const Packet c2 = preinterpret<Packet>(plogical_shift_left<MantissaBits>(padd(b_remainder, bias))); // 2^(e-3*b) - return pmul(pmul(pmul(pmul(a, c1), c1), c1), c2); // a * 2^e + const PacketI b = pcast<Packet, PacketI>(pmin(pmax(exponent, neg_last_max), last_max)); + const PacketI t = pandnot(psub(e, b), pset1<PacketI>(ScalarI(1))); // even + const Packet c1 = preinterpret<Packet>(plogical_shift_left<MantissaBits - 1>(padd(t, padd(bias, bias)))); // 2^(t/2) + const Packet c2 = preinterpret<Packet>(plogical_shift_left<MantissaBits>(padd(psub(e, t), bias))); // 2^(e-t) + return pldexp_apply_factors(a, c1, c2); // a * 2^e } // Explicitly multiplies
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h b/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h index 518695c..609d803 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h
@@ -649,20 +649,42 @@ // Sign Function //---------------------------------------------------------------------- +template <typename Packet, typename Scalar = typename unpacket_traits<Packet>::type> +struct psign_uses_binary_encoding + : bool_constant<!is_scalar<Packet>::value && packet_has_integer_packet<Packet>::value && + (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value)> {}; + template <typename Packet> -struct psign_impl<Packet, std::enable_if_t<!is_scalar<Packet>::value && +struct psign_impl<Packet, std::enable_if_t<psign_uses_binary_encoding<Packet>::value>> { + static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { + using Scalar = typename unpacket_traits<Packet>::type; + using PacketI = typename unpacket_traits<Packet>::integer_packet; + using IntScalar = typename unpacket_traits<PacketI>::type; + using Binary = binary_floating_point_traits<Scalar>; + + // Magnitude is a nonnegative integer. Comparing it with 1 avoids Clang + // folding an integer == 0 test into a DAZ-sensitive FP comparison. + const PacketI magnitude = pand(preinterpret<PacketI>(a), pset1<PacketI>(IntScalar(~Binary::kSignBit))); + const Packet is_zero = preinterpret<Packet>(pcmp_lt(magnitude, pset1<PacketI>(IntScalar(1)))); + const Packet is_nan = preinterpret<Packet>(pcmp_lt(pset1<PacketI>(IntScalar(Binary::kExponentMask)), magnitude)); + const Packet signed_one = por(pandnot(a, pabs(a)), pset1<Packet>(Scalar(1))); + return pselect(is_nan, a, pandnot(signed_one, is_zero)); + } +}; + +template <typename Packet> +struct psign_impl<Packet, std::enable_if_t<!psign_uses_binary_encoding<Packet>::value && !is_scalar<Packet>::value && !NumTraits<typename unpacket_traits<Packet>::type>::IsComplex && !NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>> { static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { using Scalar = typename unpacket_traits<Packet>::type; const Packet cst_one = pset1<Packet>(Scalar(1)); - const Packet cst_zero = pzero(a); const Packet abs_a = pabs(a); const Packet sign_mask = pandnot(a, abs_a); - const Packet nonzero_mask = pcmp_lt(cst_zero, abs_a); + const Packet zero_mask = pcmp_eq(abs_a, pzero(abs_a)); - return pselect(nonzero_mask, por(sign_mask, cst_one), abs_a); + return pselect(pisnan(a), a, pselect(zero_mask, abs_a, por(sign_mask, cst_one))); } };
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h b/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h index 0277fef..0bde042 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathFunctionsFwd.h
@@ -43,6 +43,10 @@ template <typename Packet> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pldexp_generic(const Packet& a, const Packet& exponent); +/** \internal \returns ((a * c1) * c1) * c2 in that order, for the implementations of pldexp. */ +template <typename Packet> +EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pldexp_apply_factors(const Packet& a, const Packet& c1, const Packet& c2); + // Explicitly multiplies // a * (2^e) // clamping e to the range @@ -55,6 +59,19 @@ template <typename Packet> EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Packet pldexp_fast(const Packet& a, const Packet& exponent); +namespace unary_pow { +// The two operations on a floating-point packet's raw bits that the integer-exponent pow needs: a right shift by +// the mantissa width and a lane-wise subtraction, both of the exponent field. Defined in GenericPacketMathPow.h +// in terms of unpacket_traits<Packet>::integer_packet. Override these, and has_exponent_bit_ops, where there is +// no integer_packet but the two operations can still be expressed. +template <typename Packet, typename = void> +struct has_exponent_bit_ops; +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet exponent_bits_shift_right(const Packet& bits); +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet exponent_bits_sub(const Packet& a_bits, const Packet& b_bits); +} // namespace unary_pow + /** \internal \returns cbrt(x) for single precision float */ template <typename Packet> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcbrt_float(const Packet& x_in);
diff --git a/Eigen/src/Core/arch/Default/GenericPacketMathPow.h b/Eigen/src/Core/arch/Default/GenericPacketMathPow.h index 06266e9..66d9e6e 100644 --- a/Eigen/src/Core/arch/Default/GenericPacketMathPow.h +++ b/Eigen/src/Core/arch/Default/GenericPacketMathPow.h
@@ -380,7 +380,6 @@ const Packet cst_inf = pinf<Packet>(); const Packet cst_zero = pset1<Packet>(Scalar(0)); const Packet cst_one = pset1<Packet>(Scalar(1)); - const Packet cst_nan = pnan<Packet>(); const Packet x_abs = pabs(x); Packet result = generic_pow_impl(x_abs, y); @@ -414,7 +413,9 @@ // * pow(base, exp) returns NaN if base is finite and negative // and exp is finite and non-integer. - result = pselect(pandnot(x_is_negative, y_is_int), cst_nan, result); + // A packet comparison mask is all ones, a NaN bit pattern, so por selects NaN and pand selects a constant + // against zero. Scalar masks hold the value one instead, but this overload takes packets only. + result = por(pandnot(x_is_negative, y_is_int), result); // * pow(±0, exp), where exp is negative, finite, and is an even integer or // a non-integer, returns +∞ @@ -425,7 +426,7 @@ // * pow(+0, exp), where exp is a positive odd integer, returns +0 // * pow(-0, exp), where exp is a positive odd integer, returns -0 // Sign is flipped by the rule below. - result = pselect(x_is_zero, pselect(y_is_negative, cst_inf, cst_zero), result); + result = pselect(x_is_zero, pand(y_is_negative, cst_inf), result); // pow(base, exp) returns -pow(abs(base), exp) if base has the sign bit set, // and exp is an odd integer exponent. @@ -437,7 +438,7 @@ // * pow(base, +∞) returns +∞ for any |base|>1 // * pow(±0, -∞) returns +∞ // * pow(-1, +-∞) = 1 - Packet inf_y_val = pselect(pxor(y_is_negative, x_abs_gt_one), cst_inf, cst_zero); + Packet inf_y_val = pand(pxor(y_is_negative, x_abs_gt_one), cst_inf); inf_y_val = pselect(pcmp_eq(x, pset1<Packet>(Scalar(-1.0))), cst_one, inf_y_val); result = pselect(y_abs_is_huge, inf_y_val, result); @@ -449,12 +450,12 @@ // * pow(-∞, exp) returns -∞ if exp is a positive odd integer. // * pow(-∞, exp) returns +∞ if exp is a positive non-integer or positive // even integer. - auto x_pos_inf_value = pselect(y_is_negative, cst_zero, cst_inf); + auto x_pos_inf_value = pandnot(cst_inf, y_is_negative); auto x_neg_inf_value = pselect(y_is_odd_int, pnegate(x_pos_inf_value), x_pos_inf_value); result = pselect(x_abs_is_inf, pselect(x_is_negative, x_neg_inf_value, x_pos_inf_value), result); // All cases of NaN inputs return NaN, except the two below. - result = pselect(por(pisnan(x), pisnan(y)), cst_nan, result); + result = por(por(pisnan(x), pisnan(y)), result); // * pow(base, 1) returns base. // * pow(base, +/-0) returns 1, regardless of base, even NaN. @@ -472,23 +473,24 @@ namespace unary_pow { +// Integer exponents up to this magnitude use repeated squaring; larger ones take the log/exp path of generic_pow +// where that can represent them. The crossover is where a squaring step per exponent bit stops being cheaper +// than generic_pow, whose cost per element is about four times higher for double than for float (half the +// lanes, longer polynomials): measured at 2^12 for float and 2^20 for double on AVX2 with FMA. Complex bases +// take their real scalar's value; it only limits their scalar path, since no vectorized complex pow exists. +template <typename Scalar> +constexpr numext::uint64_t max_squaring_exponent() { + return numext::uint64_t(1) << (std::is_same<typename NumTraits<Scalar>::Real, double>::value ? 20 : 12); +} + template <typename ScalarExponent, bool IsInteger = NumTraits<ScalarExponent>::IsInteger> struct exponent_helper { - using safe_abs_type = ScalarExponent; - static constexpr ScalarExponent one_half = ScalarExponent(0.5); - // these routines assume that exp is an integer stored as a floating point type - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ScalarExponent safe_abs(const ScalarExponent& exp) { - return numext::abs(exp); - } - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_odd(const ScalarExponent& exp) { + using safe_abs_type = numext::uint64_t; + // this routine assumes that exp is an integer of magnitude at most max_squaring_exponent() stored as a floating + // point type + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE safe_abs_type safe_abs(const ScalarExponent& exp) { eigen_assert(((numext::isfinite)(exp) && exp == numext::floor(exp)) && "exp must be an integer"); - ScalarExponent exp_div_2 = exp * one_half; - ScalarExponent floor_exp_div_2 = numext::floor(exp_div_2); - return exp_div_2 != floor_exp_div_2; - } - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ScalarExponent floor_div_two(const ScalarExponent& exp) { - ScalarExponent exp_div_2 = exp * one_half; - return numext::floor(exp_div_2); + return static_cast<safe_abs_type>(numext::abs(exp)); } }; @@ -502,52 +504,780 @@ safe_abs_type result = safe_abs_type(exp ^ mask); return result + safe_abs_type(ScalarExponent(1) & mask); } - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_odd(const safe_abs_type& exp) { - return exp % safe_abs_type(2) != safe_abs_type(0); - } - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE safe_abs_type floor_div_two(const safe_abs_type& exp) { - return exp >> safe_abs_type(1); +}; + +template <typename ScalarExponent, bool IsSigned = NumTraits<ScalarExponent>::IsSigned> +struct exponent_is_negative { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const ScalarExponent& exponent) { + return exponent < ScalarExponent(0); } }; -template <typename Packet, typename ScalarExponent, - bool ReciprocateIfExponentIsNegative = - !NumTraits<typename unpacket_traits<Packet>::type>::IsInteger && NumTraits<ScalarExponent>::IsSigned> -struct reciprocate { - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { - using Scalar = typename unpacket_traits<Packet>::type; - const Packet cst_pos_one = pset1<Packet>(Scalar(1)); - return exponent < 0 ? pdiv(cst_pos_one, x) : x; - } +template <typename ScalarExponent> +struct exponent_is_negative<ScalarExponent, false> { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const ScalarExponent&) { return false; } }; +// Left-to-right binary exponentiation, so that the base is multiplied in as is: each step squares the running +// power and multiplies by the base when the next exponent bit is set. +template <typename AbsExponentType> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AbsExponentType highest_set_bit(AbsExponentType m) { + AbsExponentType bit = AbsExponentType(1); + while ((m >> 1) >= bit) bit <<= 1; + return bit; +} + +// Repeated squaring for integer bases, wrapping on overflow like the underlying multiplication. template <typename Packet, typename ScalarExponent> -struct reciprocate<Packet, ScalarExponent, false> { - // pdiv not defined, nor necessary for integer base types - // if the exponent is unsigned, then the exponent cannot be negative - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent&) { return x; } -}; - -template <typename Packet, typename ScalarExponent> -EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet int_pow(const Packet& x, const ScalarExponent& exponent) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet int_pow_wrapping(const Packet& x, const ScalarExponent& exponent) { using Scalar = typename unpacket_traits<Packet>::type; using ExponentHelper = exponent_helper<ScalarExponent>; using AbsExponentType = typename ExponentHelper::safe_abs_type; - const Packet cst_pos_one = pset1<Packet>(Scalar(1)); + Packet cst_pos_one = pset1<Packet>(Scalar(1)); if (exponent == ScalarExponent(0)) return cst_pos_one; + eigen_assert(!exponent_is_negative<ScalarExponent>::run(exponent)); - Packet result = reciprocate<Packet, ScalarExponent>::run(x, exponent); - Packet y = cst_pos_one; AbsExponentType m = ExponentHelper::safe_abs(exponent); + Packet y = x; + for (AbsExponentType bit = highest_set_bit(m) >> 1; bit != 0; bit >>= 1) { + y = pmul(y, y); + if ((m & bit) != 0) y = pmul(y, x); + } + return y; +} - while (m > 1) { - bool odd = ExponentHelper::is_odd(m); - if (odd) y = pmul(y, result); - result = pmul(result, result); - m = ExponentHelper::floor_div_two(m); +// Double-word repeated squaring needs an exact two-product and the binary layout below; it serves binary32 and +// binary64 bases, real or complex. Other bases keep plain repeated squaring. +template <typename Scalar> +struct is_double_word_base + : bool_constant<(std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value) && + std::numeric_limits<Scalar>::is_iec559> {}; +template <typename RealScalar> +struct is_double_word_base<std::complex<RealScalar>> : is_double_word_base<RealScalar> {}; + +// Whether exponent_bits_shift_right/exponent_bits_sub are available, not whether integer_packet itself is: +// AVX/PacketMath.h supplies all three for Packet4d without AVX2, where there is no Packet4l. +template <typename Packet, typename> +struct has_exponent_bit_ops : false_type {}; +template <typename Packet> +struct has_exponent_bit_ops<Packet, void_t<typename unpacket_traits<Packet>::integer_packet>> : true_type {}; + +template <typename Packet, bool IsComplex = NumTraits<typename unpacket_traits<Packet>::type>::IsComplex, + bool IsScalar = is_scalar<Packet>::value> +struct real_view { + using type = Packet; +}; +template <typename Packet> +struct real_view<Packet, true, false> { + using type = typename unpacket_traits<Packet>::as_real; +}; +template <typename Scalar> +struct real_view<Scalar, true, true> { + using type = typename NumTraits<Scalar>::Real; +}; + +template <typename Packet> +struct use_double_word : bool_constant<is_double_word_base<typename unpacket_traits<Packet>::type>::value && + has_exponent_bit_ops<typename real_view<Packet>::type>::value> {}; + +// ARMv7 NEON flushes subnormal float operands and results to zero in its arithmetic and comparisons; the scalar VFP +// unit, which runs the scalar path, does not. Its dispatch is also under #if EIGEN_ARCH_ARM: elsewhere, although it +// selects the unchanged path, GCC inlines the pow kernels differently. +template <typename Packet> +struct flushes_subnormals + : bool_constant<EIGEN_ARCH_ARM && !is_scalar<Packet>::value && + std::is_same<typename NumTraits<typename unpacket_traits<Packet>::type>::Real, float>::value> {}; + +// Declared in GenericPacketMathFunctionsFwd.h so that a backend header can override them. +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet exponent_bits_shift_right(const Packet& bits) { + using PacketI = typename unpacket_traits<Packet>::integer_packet; + constexpr int kMantissaBits = numext::numeric_limits<typename unpacket_traits<Packet>::type>::digits - 1; + return preinterpret<Packet>(plogical_shift_right<kMantissaBits>(preinterpret<PacketI>(bits))); +} +template <typename Packet> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet exponent_bits_sub(const Packet& a_bits, const Packet& b_bits) { + using PacketI = typename unpacket_traits<Packet>::integer_packet; + return preinterpret<Packet>(psub(preinterpret<PacketI>(a_bits), preinterpret<PacketI>(b_bits))); +} + +// Power-of-two scaling of a real packet through its exponent bits. A finite value's magnitude is never let far +// from one, so the residuals of the double-word arithmetic stay normal (a subnormal residual costs a microcode +// assist on x86 for every operation that touches it) and no intermediate overflows or underflows. +template <typename Packet> +struct binary_exponent_scaling { + using Scalar = typename unpacket_traits<Packet>::type; + using Bits = std::make_unsigned_t<typename make_integer<Scalar>::type>; + static constexpr int kMantissaBits = numext::numeric_limits<Scalar>::digits - 1; + static constexpr int kBias = numext::numeric_limits<Scalar>::max_exponent - 1; + static constexpr Bits kExponentMask = ((Bits(1) << (CHAR_BIT * sizeof(Scalar) - kMantissaBits - 1)) - Bits(1)) + << kMantissaBits; + // 2^kMantissaBits: or-ing an integer below 2^kMantissaBits into its low bits adds that integer to it. + static constexpr Bits kMagicBits = Bits(kBias + kMantissaBits) << kMantissaBits; + + // For a normal x = m * 2^e with 2 <= |m| < 4, returns 2^-e and sets e (as a floating-point value). The + // target [2, 4) rather than [1, 2) keeps 2^-e a normal number for every x. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet inverse_scale(const Packet& x, Packet& e) { + Packet exponent_bits = pand(x, pset1frombits<Packet>(kExponentMask)); + Packet biased = exponent_bits_shift_right(exponent_bits); + Packet magic = pset1frombits<Packet>(kMagicBits); + e = psub(por(biased, magic), padd(magic, pset1<Packet>(Scalar(kBias + 1)))); + // 2^(bias + 1 - E) has biased exponent 2 * bias + 1 - E, in [1, 2 * bias] for a normal x. + Packet two_bias_plus_one = pset1frombits<Packet>(Bits(2 * kBias + 1) << kMantissaBits); + return exponent_bits_sub(two_bias_plus_one, exponent_bits); } - return pmul(y, result); + // x * 2^e for the scaled power, whose |x| lies within 2^(+-62): beyond the clamp the result is infinite or zero + // either way, and the scalar pldexp converts the exponent to int. Below kZeroBelow the result rounds to zero, + // which multiplying by zero gives exactly, whereas pldexp would underflow through several subnormal + // intermediates, each a microcode assist on x86. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet scale_result(const Packet& x, const Packet& e) { + constexpr int kLimit = 4 * numext::numeric_limits<Scalar>::max_exponent; + constexpr int kZeroBelow = + numext::numeric_limits<Scalar>::min_exponent - numext::numeric_limits<Scalar>::digits - 64; + Packet zero_below = pset1<Packet>(Scalar(kZeroBelow)); + Packet value = pselect(pcmp_lt(e, zero_below), pmul(x, pzero(x)), x); + Packet exponent = pmin(pmax(e, zero_below), pset1<Packet>(Scalar(kLimit))); +#if EIGEN_ARCH_ARM + return with_subnormals(value, exponent, pldexp(value, exponent), flushes_subnormals<Packet>()); +#else + return pldexp(value, exponent); +#endif + } + + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_subnormals(const Packet&, const Packet&, const Packet& r, + false_type) { + return r; + } + // A flushing pldexp returns zero where x * 2^e lies below the smallest normal, i.e. t = |x| * 2^(e - min_exponent + + // digits) < 2^kMantissaBits. Rounded once, as pldexp rounds, that is k * 2^(min_exponent - digits) for k = rint(t) <= + // 2^kMantissaBits, the bit pattern of its magnitude. Only lanes with |r| < min are scaled, as elsewhere the scaling + // can overflow and its conversion raise FE_INVALID, and of those only lanes with t < 2^kMantissaBits are rebuilt: + // pldexp also zeroes some normal x * 2^e through a flushed partial product (2^-124 * 2^-1 in float). + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_subnormals(const Packet& x, const Packet& e, const Packet& r, + true_type) { + using PacketI = typename unpacket_traits<Packet>::integer_packet; + constexpr int kShift = numext::numeric_limits<Scalar>::digits - numext::numeric_limits<Scalar>::min_exponent; + Packet below = pcmp_lt(pabs(r), pset1<Packet>((numext::numeric_limits<Scalar>::min)())); + if (!predux_any(below)) return r; + Packet t = pldexp(pand(below, pabs(x)), padd(e, pset1<Packet>(Scalar(kShift)))); + Packet k = preinterpret<Packet>(pcast<Packet, PacketI>(print(t))); + Packet rebuilt = pand(below, pcmp_lt(t, pset1<Packet>(Scalar(Bits(1) << kMantissaBits)))); + return pselect(rebuilt, por(k, pand(x, pset1<Packet>(Scalar(-0.0)))), r); + } + + // Whether a lane is subnormal, tested on its bits, as a flushing comparison reads a subnormal as zero. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_subnormal(const Packet& x) { + using PacketI = typename unpacket_traits<Packet>::integer_packet; + using Int = typename unpacket_traits<PacketI>::type; + PacketI magnitude = preinterpret<PacketI>(pandnot(x, pset1<Packet>(Scalar(-0.0)))); + PacketI subnormal = + pand(pcmp_lt(pzero(magnitude), magnitude), pcmp_lt(magnitude, pset1<PacketI>(Int(1) << kMantissaBits))); + return predux_any(preinterpret<Packet>(subnormal)); + } + + // Factors x = m * 2^e with 2 <= |m| < 4 for a finite, nonzero x, where m = (x * lift) * scale: lift is 2^digits + // for a subnormal x and one otherwise, and both products are exact. Zero and infinity make m NaN (their scale + // is infinite, respectively zero), which the callers resolve at the end; NaN stays NaN. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void input_scale(const Packet& x, Packet& lift, Packet& scale, + Packet& e) { + constexpr int kDigits = numext::numeric_limits<Scalar>::digits; + Packet is_subnormal = pcmp_lt(pabs(x), pset1<Packet>((numext::numeric_limits<Scalar>::min)())); + lift = pselect(is_subnormal, pset1<Packet>(Scalar(Bits(1) << kDigits)), pset1<Packet>(Scalar(1))); + scale = inverse_scale(pmul(x, lift), e); + e = psub(e, pselect(is_subnormal, pset1<Packet>(Scalar(kDigits)), pzero(x))); + } +}; + +// The running power of repeated squaring: a double word {hi, lo} times 2^exponent, with the exponent kept as a +// floating-point value that is exact while the result is finite, plus what a zero or infinite base turns into: +// itself, or its reciprocal, with the sign dropped for an even exponent. Such a base makes hi NaN from the input +// scaling on, and a finite nonzero base cannot make hi NaN or zero since the scaled power stays within 2^(+-62), +// so result() substitutes the special value exactly where hi is NaN or zero. (A zero hi would also lose its sign +// in the double-word sums.) NaN needs nothing: it propagates. +template <typename Packet, bool IsComplex = NumTraits<typename unpacket_traits<Packet>::type>::IsComplex> +struct repeated_squaring_ops { + using Scalar = typename unpacket_traits<Packet>::type; + using Scaling = binary_exponent_scaling<Packet>; + using Bound = Packet; + struct State { + Packet hi, lo, exponent, special; + }; + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_subnormal(const Packet& x) { return Scaling::any_subnormal(x); } + // Whether a finite nonzero base has a power r below the smallest normal. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_below_normal(const Packet& x, const Packet& r) { + Packet abs_x = pabs(x); + Packet regular = pand(pcmp_lt(pzero(x), abs_x), pcmp_lt(abs_x, pset1<Packet>(NumTraits<Scalar>::infinity()))); + return predux_any(pand(regular, pcmp_lt(pabs(r), pset1<Packet>((numext::numeric_limits<Scalar>::min)())))); + } + // Whether every lane lies within [1/bound, bound], where the power and its residuals stay normal without scaling. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool in_range(const Packet& x, const Packet& bound) { + Packet abs_x = pabs(x); + Packet out = por(pcmp_lt(pmul(abs_x, bound), pset1<Packet>(Scalar(1))), pcmp_lt_or_nan(bound, abs_x)); + return !predux_any(out); + } + // The double word of m or of 1/m = q + (1 - q*m)/m, where 1 - q*m is formed from the exact product + // q*m = p_hi + p_lo (1 - p_hi is exact as p_hi is within rounding of 1); renormalizing makes hi the correctly + // rounded reciprocal. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void power_base(const Packet& m, bool reciprocal, Packet& hi, + Packet& lo) { + if (!reciprocal) { + hi = m; + lo = pzero(m); + return; + } + Packet cst_pos_one = pset1<Packet>(Scalar(1)); + Packet q = pdiv(cst_pos_one, m); + Packet p_hi, p_lo; + twoprod(q, m, p_hi, p_lo); + fast_twosum(q, pdiv(psub(psub(cst_pos_one, p_hi), p_lo), m), hi, lo); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE State base(const Packet& x, bool reciprocal, bool scaled, + const Scalar&) { + State b; + if (!scaled) { + power_base(x, reciprocal, b.hi, b.lo); + b.exponent = pzero(x); + b.special = x; + return b; + } + Packet lift, scale; + Scaling::input_scale(x, lift, scale, b.exponent); + power_base(pmul(pmul(x, lift), scale), reciprocal, b.hi, b.lo); + if (reciprocal) { + b.exponent = pnegate(b.exponent); + b.special = pdiv(pset1<Packet>(Scalar(1)), x); + } else { + b.special = x; + } + return b; + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void multiply(State& y, const State& b) { + fast_twoprod(y.hi, y.lo, b.hi, b.lo, y.hi, y.lo); + y.exponent = padd(y.exponent, b.exponent); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void square(State& y) { multiply(y, y); } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void renormalize(State& y) { + Packet e; + Packet scale = Scaling::inverse_scale(y.hi, e); + y.hi = pmul(y.hi, scale); + y.lo = pmul(y.lo, scale); + y.exponent = padd(y.exponent, e); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet result(const Packet&, const State& y, const State& b, bool odd, + bool scaled) { + if (!scaled) return y.hi; + Packet use_special = por(pisnan(y.hi), pcmp_eq(y.hi, pzero(y.hi))); + return pselect(use_special, odd ? b.special : pabs(b.special), Scaling::scale_result(y.hi, y.exponent)); + } + // With a scaled |base| in [1/4, 4) a step at most cubes the magnitude bound, so four steps keep it within 2^(+-62) + // and the residuals, u times smaller, normal. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int renormalization_steps(const Scalar&) { return 4; } + // Zero, infinite and NaN bases are resolved by the special value in result(). + struct SpecialBases { + static constexpr bool any = false; + }; + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SpecialBases special_bases(const Packet&, bool) { + return SpecialBases(); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet without_special_bases(const Packet& x, const SpecialBases&) { + return x; + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_special_bases(const Packet& r, const SpecialBases&, bool) { + return r; + } +}; + +// The real and imaginary parts of a complex value or packet as two values of a real representation R, on which +// the complex algorithm below runs component-wise. For a complex packet R is its interleaved real view with both +// lanes of a pair holding the same component, so every lane operation applies to the pair at once. +template <typename Packet, bool IsScalar = is_scalar<Packet>::value> +struct complex_components { + using Scalar = typename unpacket_traits<Packet>::type; + using RealScalar = typename NumTraits<Scalar>::Real; + using R = typename unpacket_traits<Packet>::as_real; + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R flip(const R& x) { return pcplxflip(Packet(x)).v; } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R odd_lanes() { + return pcmp_eq(pset1<Packet>(Scalar(0, 1)).v, pset1<R>(RealScalar(1))); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void split(const Packet& z, R& re, R& im) { + R odd = odd_lanes(); + re = pselect(odd, flip(z.v), z.v); + im = pselect(odd, z.v, flip(z.v)); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet join(const R& re, const R& im) { + return Packet(pselect(odd_lanes(), im, re)); + } + // max(|re|, |im|) in both lanes of each pair. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R magnitude(const Packet& z) { + R abs_z = pabs(z.v); + return pmax(abs_z, flip(abs_z)); + } + // min(|re|, |im|) in both lanes of each pair. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R minor_magnitude(const Packet& z) { + R abs_z = pabs(z.v); + return pmin(abs_z, flip(abs_z)); + } + // Whether a component exceeds bound or is NaN, in its own lane: pmax need not propagate NaN. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R exceeds(const Packet& z, const R& bound) { + return pcmp_lt_or_nan(bound, pabs(z.v)); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_zero(const Packet& z) { + return predux_any(pcmp_eq(z.v, pzero(z.v))); + } + // In both lanes of each pair: both components zero; a component infinite; a component NaN and none infinite. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void special_masks(const Packet& z, R& zero, R& inf, R& nan) { + R zero_lane = pcmp_eq(z.v, pzero(z.v)); + R inf_lane = pcmp_eq(pabs(z.v), pset1<R>(NumTraits<RealScalar>::infinity())); + R nan_lane = pisnan(z.v); + zero = pand(zero_lane, flip(zero_lane)); + inf = por(inf_lane, flip(inf_lane)); + nan = pandnot(por(nan_lane, flip(nan_lane)), inf); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet replace(const R& mask, const Scalar& value, const Packet& z) { + return Packet(pselect(mask, pset1<Packet>(value).v, z.v)); + } +}; + +template <typename Scalar> +struct complex_components<Scalar, true> { + using R = typename NumTraits<Scalar>::Real; + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void split(const Scalar& z, R& re, R& im) { + re = numext::real(z); + im = numext::imag(z); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar join(const R& re, const R& im) { return Scalar(re, im); } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R magnitude(const Scalar& z) { + return numext::maxi(numext::abs(numext::real(z)), numext::abs(numext::imag(z))); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R minor_magnitude(const Scalar& z) { + return numext::mini(numext::abs(numext::real(z)), numext::abs(numext::imag(z))); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R exceeds(const Scalar& z, const R& bound) { + return por(pcmp_lt_or_nan(bound, numext::abs(numext::real(z))), + pcmp_lt_or_nan(bound, numext::abs(numext::imag(z)))); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_zero(const Scalar& z) { + return numext::real(z) == R(0) || numext::imag(z) == R(0); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void special_masks(const Scalar& z, R& zero, R& inf, R& nan) { + R re = numext::real(z), im = numext::imag(z); + bool is_inf = (numext::isinf)(re) || (numext::isinf)(im); + zero = re == R(0) && im == R(0) ? R(1) : R(0); + inf = is_inf ? R(1) : R(0); + nan = !is_inf && ((numext::isnan)(re) || (numext::isnan)(im)) ? R(1) : R(0); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar replace(const R& mask, const Scalar& value, const Scalar& z) { + return mask != R(0) ? value : z; + } +}; + +// Complex bases keep the real and imaginary parts as separate double words sharing one exponent, scaled by the +// larger component: z^2 = (a^2 - b^2) + 2ab i is three products and a product (a + bi)(c + di) four. Either +// component may legitimately be zero. Zero and infinite bases are the two points 0 and infinity of the extended +// complex plane, as std::proj sees it: every value with an infinite component, a NaN one included, is the same +// infinity, +inf + 0i. So a zero base gives +0 + 0i for n > 0 and +inf + 0i for n < 0, an infinite base the reverse, +// whatever the signs of their zeros; a base with a NaN component and no infinite one gives NaN + NaN i. std::pow +// leaves all of these to the implementation, and libstdc++'s follow its evaluation order: 1/0 = inf + nan i, +// (-inf + 2i)^3 = nan + inf i. +template <typename Packet> +struct repeated_squaring_ops<Packet, true> { + using Scalar = typename unpacket_traits<Packet>::type; + using Real = typename NumTraits<Scalar>::Real; + using Components = complex_components<Packet>; + using R = typename Components::R; + using Scaling = binary_exponent_scaling<R>; + using Bound = R; + // Lanes marked `separated` hold only the larger component L (a, or bi) of z = L (1 + delta); the power is + // L^n (1 + i c) with i c = n delta, c = (coefficient_hi + coefficient_lo) 2^coefficient_exponent (see base()). + struct State { + R re_hi, re_lo, im_hi, im_lo, exponent; + R separated, coefficient_hi, coefficient_lo, coefficient_exponent; + bool any_separated; + bool negative, any_zero; + }; + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_subnormal(const Packet& x) { + return Scaling::any_subnormal(x.v); + } + // A complex power has no single-operation form for the scalar path to round differently. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_below_normal(const Packet&, const Packet&) { return false; } + // Four steps between renormalizations let the power fall to 2^-62 of the scale when the base is a reciprocal in + // [1/4, 1/2), and the smaller component, of order r = |s/L| times the power, keeps normal residuals only for + // r >= 2^(min_exponent - 1 + digits + 62). Below that the base is separated (see base()), and first order is + // exact to within (n r)^2 / 2 <= u/4 for |n| <= 2^kFirstOrderLog2Limit: every n for double, 2^25 for float. + // Larger n renormalize at every step instead, which keeps the power above 2^-6 (a step from [2, 4), or from the + // base itself) and lets the separation drop 56 binades, where first order holds for any n. + static constexpr int kMinExponent = numext::numeric_limits<Real>::min_exponent; + static constexpr int kDigits = numext::numeric_limits<Real>::digits; + static constexpr int kSeparationExponent = kMinExponent + kDigits + 64; + static constexpr int kFirstOrderLog2Limit = (1 - kDigits - 2 * kSeparationExponent) / 2; + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool every_step(const Real& count) { + return kFirstOrderLog2Limit < 64 && + count > Real(numext::uint64_t(1) << (kFirstOrderLog2Limit < 64 ? kFirstOrderLog2Limit : 0)); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int renormalization_steps(const Real& count) { + return every_step(count) ? 1 : 4; + } + // Whether every lane's max(|re|, |im|), and min(|re|, |im|) unless zero, lie within [1/bound, bound], where the + // power and its residuals stay normal without scaling. A zero component stays exactly zero in every power. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool in_range(const Packet& x, const R& bound) { + R magnitude = Components::magnitude(x); + R minor = Components::minor_magnitude(x); + R one = pset1<R>(Real(1)); + R out = por(pcmp_lt(pmul(magnitude, bound), one), Components::exceeds(x, bound)); + out = por(out, pandnot(pcmp_lt(pmul(minor, bound), one), pcmp_eq(minor, pzero(minor)))); + return predux_any(out) == false; + } + template <typename Count> + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE State base(const Packet& x, bool reciprocal, bool scaled, + const Count& m) { + State b; + b.any_separated = false; + R re, im; + Components::split(x, re, im); + b.negative = reciprocal; + b.any_zero = Components::any_zero(x); + if (!scaled) { + power_base(re, im, reciprocal, b); + b.exponent = pzero(re); + return b; + } + R lift, scale; + Scaling::input_scale(Components::magnitude(x), lift, scale, b.exponent); + R wr = pmul(pmul(re, lift), scale), wi = pmul(pmul(im, lift), scale); + // Both components take the larger one's scale, under which a nonzero smaller component s far below the larger + // one L would lose its residuals (see kSeparationExponent). Then z = L (1 + delta) with |delta| tiny and + // z^n = L^n (1 + n delta) to working precision: the power runs on L alone and result() adds L^n n delta, with + // delta = i b/a, or a/(bi) = -i a/b, and -delta for the reciprocal base. + int separation_exponent = every_step(Real(m)) ? kSeparationExponent - 56 : kSeparationExponent; + R separation = + pset1frombits<R>(typename Scaling::Bits(Scaling::kBias + separation_exponent) << Scaling::kMantissaBits); + R zero = pzero(wr); + R re_separated = pandnot(pcmp_lt(pabs(wr), separation), pcmp_eq(re, zero)); + R im_separated = pandnot(pcmp_lt(pabs(wi), separation), pcmp_eq(im, zero)); + b.separated = por(re_separated, im_separated); + b.any_separated = predux_any(b.separated); + if (b.any_separated) { + R s_lift, s_scale, s_exponent, count_exponent; + Scaling::input_scale(pselect(im_separated, im, re), s_lift, s_scale, s_exponent); + R s = pmul(pmul(pselect(im_separated, im, re), s_lift), s_scale); + // The count n is a double word too, as Real(n) is inexact beyond 2^digits: its leading digits, which Real + // holds exactly, and the rest. + numext::uint64_t n = numext::uint64_t(m); + numext::uint64_t unit = highest_set_bit(n) >> (kDigits - 1); + numext::uint64_t n_hi = unit > 1 ? n & ~(unit - 1) : n; + R count_hi = pset1<R>(Real(n_hi)), count_lo = pset1<R>(Real(n - n_hi)); + R count_scale = Scaling::inverse_scale(count_hi, count_exponent); + // The coefficient n s / L is a double word so that result() rounds the added component once: the ratio + // q + (s - q L) / L, where s - q L = (s - p_hi) - p_lo is exact, times the count. The scaled |s / L| is in + // (1/2, 2) and the scaled count in [2, 4); an eighth keeps |c hi| within 2^62. + R l = pselect(im_separated, wr, wi); + R q = pdiv(s, l); + R p_hi, p_lo, c_hi, c_lo; + twoprod(q, l, p_hi, p_lo); + R q_lo = pdiv(psub(psub(s, p_hi), p_lo), l); + R scaled_count = pmul(count_hi, count_scale); + R scaled_count_lo = pmul(count_lo, count_scale); + twoprod(scaled_count, q, c_hi, c_lo); + fast_twosum(c_hi, pmadd(scaled_count, q_lo, pmadd(scaled_count_lo, q, c_lo)), p_hi, p_lo); + R sign = pselect(im_separated, pset1<R>(Real(reciprocal ? -0.125 : 0.125)), + pset1<R>(Real(reciprocal ? 0.125 : -0.125))); + b.coefficient_hi = pmul(p_hi, sign); + b.coefficient_lo = pmul(p_lo, sign); + b.coefficient_exponent = padd(padd(psub(s_exponent, b.exponent), count_exponent), pset1<R>(Real(3))); + wr = pselect(re_separated, zero, wr); + wi = pselect(im_separated, zero, wi); + } + power_base(wr, wi, reciprocal, b); + if (reciprocal) b.exponent = pnegate(b.exponent); + return b; + } + // The double words of w or of 1/w = q + e*q with q = conj(w)/|w|^2 and e = 1 - q*w = er - t i. The norm cannot + // over- or underflow for max(|wr|, |wi|) in [2, 4) or within the unscaled range, and the few ulps of error in q + // are what the residual corrects: e is of order u and is formed from exact products so that e*q carries the + // correction to order u^2. 1 - s_hi is exact as s_hi is within rounding of 1. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void power_base(const R& wr, const R& wi, bool reciprocal, State& b) { + b.re_lo = b.im_lo = pzero(wr); + if (!reciprocal) { + b.re_hi = wr; + b.im_hi = wi; + return; + } + R inv = pdiv(pset1<R>(typename NumTraits<Scalar>::Real(1)), pmadd(wr, wr, pmul(wi, wi))); + R qr = pmul(wr, inv), qi = pnegate(pmul(wi, inv)); + R a_hi, a_lo, c_hi, c_lo, s_hi, s_lo, t_hi, t_lo; + twoprod(qr, wr, a_hi, a_lo); + twoprod(qi, wi, c_hi, c_lo); + twodiff(a_hi, a_lo, c_hi, c_lo, s_hi, s_lo); + twoprod(qr, wi, a_hi, a_lo); + twoprod(qi, wr, c_hi, c_lo); + twosum(a_hi, a_lo, c_hi, c_lo, t_hi, t_lo); + R er = psub(psub(pset1<R>(typename NumTraits<Scalar>::Real(1)), s_hi), s_lo); + R t = padd(t_hi, t_lo); + fast_twosum(qr, pmadd(er, qr, pmul(t, qi)), b.re_hi, b.re_lo); + fast_twosum(qi, pmsub(er, qi, pmul(t, qr)), b.im_hi, b.im_lo); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void square(State& y) { + R a_hi, a_lo, c_hi, c_lo, p_hi, p_lo; + fast_twoprod(y.re_hi, y.re_lo, y.re_hi, y.re_lo, a_hi, a_lo); + fast_twoprod(y.im_hi, y.im_lo, y.im_hi, y.im_lo, c_hi, c_lo); + fast_twoprod(y.re_hi, y.re_lo, y.im_hi, y.im_lo, p_hi, p_lo); + twodiff(a_hi, a_lo, c_hi, c_lo, y.re_hi, y.re_lo); + y.im_hi = padd(p_hi, p_hi); + y.im_lo = padd(p_lo, p_lo); + y.exponent = padd(y.exponent, y.exponent); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void multiply(State& y, const State& b) { + R ac_hi, ac_lo, bd_hi, bd_lo, ad_hi, ad_lo, bc_hi, bc_lo; + fast_twoprod(y.re_hi, y.re_lo, b.re_hi, b.re_lo, ac_hi, ac_lo); + fast_twoprod(y.im_hi, y.im_lo, b.im_hi, b.im_lo, bd_hi, bd_lo); + fast_twoprod(y.re_hi, y.re_lo, b.im_hi, b.im_lo, ad_hi, ad_lo); + fast_twoprod(y.im_hi, y.im_lo, b.re_hi, b.re_lo, bc_hi, bc_lo); + twodiff(ac_hi, ac_lo, bd_hi, bd_lo, y.re_hi, y.re_lo); + twosum(ad_hi, ad_lo, bc_hi, bc_lo, y.im_hi, y.im_lo); + y.exponent = padd(y.exponent, b.exponent); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void renormalize(State& y) { + R e; + R scale = Scaling::inverse_scale(pmax(pabs(y.re_hi), pabs(y.im_hi)), e); + y.re_hi = pmul(y.re_hi, scale); + y.re_lo = pmul(y.re_lo, scale); + y.im_hi = pmul(y.im_hi, scale); + y.im_lo = pmul(y.im_lo, scale); + y.exponent = padd(y.exponent, e); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet result(const Packet& x, const State& y, const State& b, bool odd, + bool scaled) { + if (!scaled) { + R re = y.re_hi, im = y.im_hi; + if (b.any_zero) zero_component_signs(x, b.negative, odd, re, im); + return Components::join(re, im); + } + R re = Scaling::scale_result(y.re_hi, y.exponent); + R im = Scaling::scale_result(y.im_hi, y.exponent); + if (b.any_separated) { + // L^n is real or imaginary, and i c L^n fills the component it leaves exactly zero. + R e = padd(y.exponent, b.coefficient_exponent); + R zero = pzero(re); + R re_hi, re_lo, im_hi, im_lo; + fast_twoprod(b.coefficient_hi, b.coefficient_lo, y.im_hi, y.im_lo, re_hi, re_lo); + fast_twoprod(b.coefficient_hi, b.coefficient_lo, y.re_hi, y.re_lo, im_hi, im_lo); + re = pselect(pand(b.separated, pcmp_eq(y.re_hi, zero)), Scaling::scale_result(pnegate(re_hi), e), re); + im = pselect(pand(b.separated, pcmp_eq(y.im_hi, zero)), Scaling::scale_result(im_hi, e), im); + } + if (b.any_zero) zero_component_signs(x, b.negative, odd, re, im); + return Components::join(re, im); + } + // Zero, infinite and NaN bases are replaced by 1 + i before the loop, so that they neither force the scaled loop on + // their packet nor come back NaN, and their results are set at the end (see the comment above the struct). 1 + i + // has no zero component, which would take the packet through zero_component_signs. Every one of them fails + // in_range, so only a packet bound for the scaled loop is checked, and only whether there are any is carried through + // the loop. + struct SpecialBases { + Packet x; + bool any; + }; + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SpecialBases special_bases(const Packet& x, bool scaled) { + SpecialBases special; + special.x = x; + special.any = false; + if (scaled) { + R zero, inf, nan; + Components::special_masks(x, zero, inf, nan); + special.any = predux_any(por(por(zero, inf), nan)); + } + return special; + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet without_special_bases(const Packet& x, + const SpecialBases& special) { + if (!special.any) return x; + R zero, inf, nan; + Components::special_masks(x, zero, inf, nan); + return Components::replace(por(por(zero, inf), nan), Scalar(1, 1), x); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_special_bases(const Packet& r, const SpecialBases& special, + bool negative) { + if (!special.any) return r; + R zero, inf, nan; + Components::special_masks(special.x, zero, inf, nan); + Real real_inf = NumTraits<Real>::infinity(), real_nan = NumTraits<Real>::quiet_NaN(); + Packet out = Components::replace(negative ? zero : inf, Scalar(real_inf, Real(0)), r); + out = Components::replace(negative ? inf : zero, Scalar(0), out); + return Components::replace(nan, Scalar(real_nan, real_nan), out); + } + // A base with one component s = +-0 is the limit of L (1 + i t), with t = s/L for a real L and -s/b for L = bi, + // so z^n = L^n (1 + i n t): the zero component of the power has the sign of i n t L^n, which the double-word sums + // lose (-0 + 0 = +0). Sign bits combine by xor; a scalar comparison mask is one rather than all ones, so masks + // select. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void zero_component_signs(const Packet& x, bool negative, bool odd, + R& re, R& im) { + R zero = pzero(re); + R x_re, x_im; + Components::split(x, x_re, x_im); + R imaginary = pcmp_eq(x_re, zero); + R one_zero = pxor(imaginary, pcmp_eq(x_im, zero)); + R minus_zero = pset1<R>(Real(-0.0)); + // i^2 = -1 from t for L = bi and from i t L^n for an imaginary power cancel for odd n. + R power_imaginary = odd ? imaginary : zero; + R n_sign = negative ? minus_zero : zero; + R extra = odd ? n_sign : pselect(imaginary, pxor(n_sign, minus_zero), n_sign); + R sign = pxor(pand(pxor(pxor(x_re, x_im), pselect(power_imaginary, im, re)), minus_zero), extra); + re = pselect(pand(one_zero, power_imaginary), sign, re); + im = pselect(pandnot(one_zero, power_imaginary), sign, im); + } +}; + +// Plain repeated squaring for the remaining floating-point bases. +template <typename Packet, typename ScalarExponent> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_plain(const Packet& x, const ScalarExponent& exponent) { + using Scalar = typename unpacket_traits<Packet>::type; + using ExponentHelper = exponent_helper<ScalarExponent>; + using AbsExponentType = typename ExponentHelper::safe_abs_type; + if (exponent == ScalarExponent(0)) return pset1<Packet>(Scalar(1)); + + Packet base = exponent_is_negative<ScalarExponent>::run(exponent) ? pdiv(pset1<Packet>(Scalar(1)), x) : x; + AbsExponentType m = ExponentHelper::safe_abs(exponent); + Packet y = base; + for (AbsExponentType bit = highest_set_bit(m) >> 1; bit != 0; bit >>= 1) { + y = pmul(y, y); + if ((m & bit) != 0) y = pmul(y, base); + } + return y; +} + +// Repeated squaring for a binary32 or binary64 base, real or complex, carried in double-word arithmetic. Plain +// repeated squaring doubles the accumulated rounding error at every squaring, so its error grows like n * u for +// x^n. Keeping the running power as an unevaluated sum {hi, lo} bounds each step's relative error by about +// 7 * u^2 (fast_twoprod), so the result is correctly rounded unless the exact power lies within about 7 * n * u^2 +// of a rounding boundary, and stays within 1 ulp up to n = max_squaring_exponent() for float. For a complex base +// the bound is normwise, relative to |z^n|, as a^2 - b^2 and ac - bd can cancel, and the n u^2 term passes u/2 +// as |n| approaches 1/u: (0.6 + 0.8i)^n in complex<float> is 0.2 u off at n = 4096, 0.5 u at 2^24 and 4.2 u at +// 2^31 - 1. The power is scaled by powers of two throughout and only the final pldexp can overflow or underflow. +template <typename Packet, typename ScalarExponent> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_double_word(const Packet& x, const ScalarExponent& exponent) { + using Scalar = typename unpacket_traits<Packet>::type; + using ExponentHelper = exponent_helper<ScalarExponent>; + using AbsExponentType = typename ExponentHelper::safe_abs_type; + using Ops = repeated_squaring_ops<Packet>; + if (exponent == ScalarExponent(0)) return pset1<Packet>(Scalar(1)); + + bool negative = exponent_is_negative<ScalarExponent>::run(exponent); + AbsExponentType m = ExponentHelper::safe_abs(exponent); + bool odd = (m & AbsExponentType(1)) != 0; + if (m == AbsExponentType(1) && !negative) return x; + // A real x^-1 and x^2 are a single correctly rounded operation, which is what the double word would produce. On + // ARMv7 NEON pdiv is a refined reciprocal estimate instead. + EIGEN_IF_CONSTEXPR (!NumTraits<Scalar>::IsComplex) { + if (m == AbsExponentType(1) && !flushes_subnormals<Packet>::value) return pdiv(pset1<Packet>(Scalar(1)), x); + if (m == AbsExponentType(2) && !negative) return pmul(x, x); + } + AbsExponentType top = highest_set_bit(m); + + // Bases whose magnitude lies within 2^(+-B) for B = budget / |n| keep every power up to x^n and its residuals + // normal: |x^n| <= 2^budget < max and u * |x^n| >= 2^(min_exponent - 1), the smallest normal, so the loop + // needs neither scaling nor exponents and the result is hi itself. A complex base is tested on both nonzero + // components, as every term a^k b^(n-k) of its power must stay normal too; its magnitude exceeds the larger one + // by up to sqrt(2), so B is one less to absorb sqrt(2)^n (an exhausted budget therefore admits nothing), and its + // reciprocal forms |w|^2, which bounds B by half the exponent range. Zero, infinity and NaN fail the test. + using Real = typename NumTraits<Scalar>::Real; + using RealBits = std::make_unsigned_t<typename make_integer<Real>::type>; + constexpr int kBudget = -(numext::numeric_limits<Real>::min_exponent + numext::numeric_limits<Real>::digits); + constexpr int kMantissaBits = numext::numeric_limits<Real>::digits - 1; + constexpr RealBits kBiasBits = RealBits(numext::numeric_limits<Real>::max_exponent - 1); + int b = numext::uint64_t(m) > numext::uint64_t(kBudget) ? 0 : kBudget / int(m); + EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) + b = numext::mini(b - 1, (numext::numeric_limits<Real>::max_exponent - 1) / 2); + // A flushing packet always scales: the budget keeps the unscaled power normal, but not all of its residuals, nor the + // components and the norm of a complex reciprocal. + EIGEN_IF_CONSTEXPR (flushes_subnormals<Packet>::value) b = -1; + typename Ops::Bound bound = + pset1frombits<typename Ops::Bound>(RealBits(kBiasBits + RealBits(b < 0 ? 0 : b)) << kMantissaBits); + bool scaled = b < 0 || !Ops::in_range(x, bound); + // Zero, infinite and NaN bases are never in range, so only a packet bound for the scaled loop can hold one. + typename Ops::SpecialBases special = Ops::special_bases(x, scaled); + Packet x_regular = Ops::without_special_bases(x, special); + if (special.any) scaled = b < 0 || !Ops::in_range(x_regular, bound); + + typename Ops::State base = Ops::base(x_regular, negative, scaled, m); + typename Ops::State y = base; + int renormalization_steps = Ops::renormalization_steps(Real(m)); + int steps_since_renormalization = 0; + for (AbsExponentType bit = top >> 1; bit != 0; bit >>= 1) { + Ops::square(y); + if ((m & bit) != 0) Ops::multiply(y, base); + if (scaled && ++steps_since_renormalization == renormalization_steps) { + Ops::renormalize(y); + steps_since_renormalization = 0; + } + } + return Ops::with_special_bases(Ops::result(x_regular, y, base, odd, scaled), special, negative); +} + +template <typename Packet, typename ScalarExponent> +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE Packet int_pow_lanewise(const Packet& x, const ScalarExponent& exponent) { + using Scalar = typename unpacket_traits<Packet>::type; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar values[unpacket_traits<Packet>::size]; + pstore(values, x); + for (Scalar& value : values) value = int_pow_double_word(value, exponent); + return pload<Packet>(values); +} + +template <typename Packet, typename ScalarExponent> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_double_word(const Packet& x, const ScalarExponent& exponent, + false_type) { + return int_pow_double_word(x, exponent); +} + +// A flushing packet is computed lane by lane on scalars, whose arithmetic does not flush, where it holds a subnormal +// base, which the comparisons, magnitudes and scalings of the base would all read as zero, or where a real x^-1 or x^2 +// falls below the smallest normal: the scalar path rounds those once, as 1/x and x * x do. +template <typename Packet, typename ScalarExponent> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_double_word(const Packet& x, const ScalarExponent& exponent, + true_type) { + using Ops = repeated_squaring_ops<Packet>; + if (Ops::any_subnormal(x)) return int_pow_lanewise(x, exponent); + Packet r = int_pow_double_word(x, exponent); + if (exponent_helper<ScalarExponent>::safe_abs(exponent) <= 2 && Ops::any_below_normal(x, r)) + return int_pow_lanewise(x, exponent); + return r; +} + +template <typename Packet, typename ScalarExponent> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow(const Packet& x, const ScalarExponent& exponent, true_type) { +#if EIGEN_ARCH_ARM + return int_pow_double_word(x, exponent, flushes_subnormals<Packet>()); +#else + return int_pow_double_word(x, exponent); +#endif +} + +template <typename Packet, typename ScalarExponent> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow(const Packet& x, const ScalarExponent& exponent, false_type) { + return int_pow_plain(x, exponent); +} + +template <typename Packet, typename ScalarExponent> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow(const Packet& x, const ScalarExponent& exponent) { + return int_pow(x, exponent, use_double_word<Packet>()); +} + +// The largest integer-valued floating-point exponent that repeated squaring handles; beyond it generic_pow takes +// over. Plain squaring is accurate to a few ulps only for small exponents. +template <typename Packet, typename ScalarExponent> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool use_repeated_squaring(const ScalarExponent& exponent) { + using Scalar = typename unpacket_traits<Packet>::type; + return use_double_word<Packet>::value ? numext::abs(exponent) <= ScalarExponent(max_squaring_exponent<Scalar>()) + : (exponent <= ScalarExponent(7) && exponent >= ScalarExponent(-3)); +} + +// The same for an exponent of integer type, which repeated squaring handles exactly whatever its size: generic_pow +// only takes over where the exponent converts exactly to the base type. +template <typename Packet, typename ScalarExponent> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool use_repeated_squaring_for_integer(const ScalarExponent& exponent) { + using Scalar = typename unpacket_traits<Packet>::type; + constexpr numext::uint64_t kExactLimit = numext::uint64_t(1) << numext::numeric_limits<Scalar>::digits; + return exponent_helper<ScalarExponent>::safe_abs(exponent) > kExactLimit || + use_repeated_squaring<Packet>(static_cast<Scalar>(exponent)); } template <typename Packet> @@ -567,6 +1297,7 @@ // Handle special cases for pow(x, exponent) where both base and exponent are // floating point and the exponent is a non-integer scalar (uniform across all // SIMD lanes). This allows us to use scalar branches on exponent properties. +// Reached from packetOp only, so the comparison masks are all ones and select a NaN or an infinity bitwise. template <typename Packet, typename ScalarExponent> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_nonint_nonint_errors(const Packet& x, const Packet& powx, const ScalarExponent& exponent) { @@ -579,7 +1310,7 @@ const Packet abs_x = pabs(x); // x < 0 with non-integer exponent -> NaN. - Packet result = pselect(pcmp_lt(x, cst_zero), cst_nan, powx); + Packet result = por(pcmp_lt(x, cst_zero), powx); if (!(numext::isfinite)(exponent)) { if (exponent != exponent) { @@ -590,10 +1321,10 @@ const Packet abs_x_is_one = pcmp_eq(abs_x, cst_one); if (exponent > ScalarExponent(0)) { // pow(x, +inf): |x| > 1 -> +inf, |x| < 1 -> 0, |x| == 1 -> 1. - result = pselect(pcmp_lt(cst_one, abs_x), cst_inf, cst_zero); + result = pand(pcmp_lt(cst_one, abs_x), cst_inf); } else { // pow(x, -inf): |x| < 1 -> +inf, |x| > 1 -> 0, |x| == 1 -> 1. - result = pselect(pcmp_lt(abs_x, cst_one), cst_inf, cst_zero); + result = pand(pcmp_lt(abs_x, cst_one), cst_inf); } // pow(+-1, +-inf) = 1. result = pselect(abs_x_is_one, cst_one, result); @@ -616,7 +1347,7 @@ // NaN base produces NaN. This overrides all cases above, but pow(NaN, 0) = 1 // and pow(NaN, integer) are handled by the integer exponent path and never // reach this function. - result = pselect(pisnan(x), cst_nan, result); + result = por(pisnan(x), result); return result; } @@ -630,7 +1361,7 @@ // This routine handles negative exponents. // The return value is either 0, 1, or -1. - const Packet cst_pos_one = pset1<Packet>(Scalar(1)); + Packet cst_pos_one = pset1<Packet>(Scalar(1)); const bool exponent_is_odd = exponent % ScalarExponent(2) != ScalarExponent(0); const Packet exp_is_odd = exponent_is_odd ? ptrue<Packet>(x) : pzero<Packet>(x); @@ -672,15 +1403,11 @@ template <typename Packet, typename ScalarExponent, bool ExponentIsSigned> struct unary_pow_impl<Packet, ScalarExponent, false, false, ExponentIsSigned> { using Scalar = typename unpacket_traits<Packet>::type; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { const bool exponent_is_integer = (numext::isfinite)(exponent) && numext::round(exponent) == exponent; if (exponent_is_integer) { - // The simple recursive doubling implementation is only accurate to 3 ulps - // for integer exponents in [-3:7]. Since this is a common case, we - // specialize it here. - bool use_repeated_squaring = - (exponent <= ScalarExponent(7) && (!ExponentIsSigned || exponent >= ScalarExponent(-3))); - return use_repeated_squaring ? unary_pow::int_pow(x, exponent) : generic_pow(x, pset1<Packet>(exponent)); + return unary_pow::use_repeated_squaring<Packet>(exponent) ? unary_pow::int_pow(x, exponent) + : generic_pow(x, pset1<Packet>(exponent)); } else { Packet result = unary_pow::gen_pow(x, exponent); result = unary_pow::handle_nonint_nonint_errors(x, result, exponent); @@ -692,7 +1419,20 @@ template <typename Packet, typename ScalarExponent, bool ExponentIsSigned> struct unary_pow_impl<Packet, ScalarExponent, false, true, ExponentIsSigned> { using Scalar = typename unpacket_traits<Packet>::type; - static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { + // Only real float and double bases with double-word support fall back to generic_pow for large exponents: + // complex bases have no vectorized generic_pow, and the bases left without double-word support (half, + // bfloat16, the GPU packets, the MSA and HVX float packets) have no packet generic_pow either, so their + // squaring loop -- at most 64 steps -- is all there is. + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { + return run(x, exponent, + bool_constant < unary_pow::use_double_word<Packet>::value && !NumTraits<Scalar>::IsComplex > ()); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(const Packet& x, const ScalarExponent& exponent, true_type) { + return unary_pow::use_repeated_squaring_for_integer<Packet>(exponent) + ? unary_pow::int_pow(x, exponent) + : generic_pow(x, pset1<Packet>(static_cast<Scalar>(exponent))); + } + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(const Packet& x, const ScalarExponent& exponent, false_type) { return unary_pow::int_pow(x, exponent); } }; @@ -704,7 +1444,7 @@ if (exponent < ScalarExponent(0)) { return unary_pow::handle_negative_exponent(x, exponent); } else { - return unary_pow::int_pow(x, exponent); + return unary_pow::int_pow_wrapping(x, exponent); } } }; @@ -713,7 +1453,7 @@ struct unary_pow_impl<Packet, ScalarExponent, true, true, false> { using Scalar = typename unpacket_traits<Packet>::type; static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) { - return unary_pow::int_pow(x, exponent); + return unary_pow::int_pow_wrapping(x, exponent); } };
diff --git a/Eigen/src/Core/arch/MSA/Complex.h b/Eigen/src/Core/arch/MSA/Complex.h index ea0d76c..d35884f 100644 --- a/Eigen/src/Core/arch/MSA/Complex.h +++ b/Eigen/src/Core/arch/MSA/Complex.h
@@ -117,6 +117,7 @@ template <> struct unpacket_traits<Packet2cf> { typedef std::complex<float> type; + typedef Packet4f as_real; enum { size = 2, alignment = Aligned16, @@ -412,6 +413,7 @@ template <> struct unpacket_traits<Packet1cd> { typedef std::complex<double> type; + typedef Packet2d as_real; enum { size = 1, alignment = Aligned16,
diff --git a/Eigen/src/Core/arch/NEON/Complex.h b/Eigen/src/Core/arch/NEON/Complex.h index f4a1c13..9c9ef80 100644 --- a/Eigen/src/Core/arch/NEON/Complex.h +++ b/Eigen/src/Core/arch/NEON/Complex.h
@@ -203,16 +203,19 @@ #else template <> EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b) { - Packet4f v1, v2; - - // Get the real values of a | a1_re | a1_re | a2_re | a2_re | - v1 = vcombine_f32(vdup_lane_f32(vget_low_f32(a.v), 0), vdup_lane_f32(vget_high_f32(a.v), 0)); - // Get the imag values of a | a1_im | a1_im | a2_im | a2_im | - v2 = vcombine_f32(vdup_lane_f32(vget_low_f32(a.v), 1), vdup_lane_f32(vget_high_f32(a.v), 1)); + // a_re = | a1_re | a1_re | a2_re | a2_re |, a_im = | a1_im | a1_im | a2_im | a2_im | +#if EIGEN_ARCH_ARM64 + Packet4f a_re = vtrn1q_f32(a.v, a.v); + Packet4f a_im = vtrn2q_f32(a.v, a.v); +#else + // ARMv7 vtrn overwrites both operands, so vtrnq_f32(a, a) costs a copy and measured slower than two vdup. + Packet4f a_re = vcombine_f32(vdup_lane_f32(vget_low_f32(a.v), 0), vdup_lane_f32(vget_high_f32(a.v), 0)); + Packet4f a_im = vcombine_f32(vdup_lane_f32(vget_low_f32(a.v), 1), vdup_lane_f32(vget_high_f32(a.v), 1)); +#endif // Multiply the real a with b - v1 = vmulq_f32(v1, b.v); + Packet4f v1 = vmulq_f32(a_re, b.v); // Multiply the imag a with b - v2 = vmulq_f32(v2, b.v); + Packet4f v2 = vmulq_f32(a_im, b.v); // Conjugate v2 v2 = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v2), p4ui_CONJ_XOR())); // Swap real/imag elements in v2. @@ -331,31 +334,26 @@ } template <> -EIGEN_DEVICE_FUNC inline Packet1cf pgather<std::complex<float>, Packet1cf>(const std::complex<float>* from, - Index stride) { - const Packet2f tmp = vdup_n_f32(std::real(from[0 * stride])); - return Packet1cf(vset_lane_f32(std::imag(from[0 * stride]), tmp, 1)); +EIGEN_DEVICE_FUNC inline Packet1cf pgather<std::complex<float>, Packet1cf>(const std::complex<float>* from, Index) { + return ploadu<Packet1cf>(from); } template <> EIGEN_DEVICE_FUNC inline Packet2cf pgather<std::complex<float>, Packet2cf>(const std::complex<float>* from, Index stride) { - Packet4f res = vdupq_n_f32(std::real(from[0 * stride])); - res = vsetq_lane_f32(std::imag(from[0 * stride]), res, 1); - res = vsetq_lane_f32(std::real(from[1 * stride]), res, 2); - res = vsetq_lane_f32(std::imag(from[1 * stride]), res, 3); - return Packet2cf(res); + return Packet2cf(vcombine_f32(vld1_f32(reinterpret_cast<const float*>(from)), + vld1_f32(reinterpret_cast<const float*>(from + stride)))); } template <> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet1cf>(std::complex<float>* to, const Packet1cf& from, - Index stride) { - to[stride * 0] = std::complex<float>(vget_lane_f32(from.v, 0), vget_lane_f32(from.v, 1)); + Index) { + pstoreu(to, from); } template <> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet2cf>(std::complex<float>* to, const Packet2cf& from, Index stride) { - to[stride * 0] = std::complex<float>(vgetq_lane_f32(from.v, 0), vgetq_lane_f32(from.v, 1)); - to[stride * 1] = std::complex<float>(vgetq_lane_f32(from.v, 2), vgetq_lane_f32(from.v, 3)); + vst1_f32(reinterpret_cast<float*>(to), vget_low_f32(from.v)); + vst1_f32(reinterpret_cast<float*>(to + stride), vget_high_f32(from.v)); } template <> @@ -378,7 +376,7 @@ template <> EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { - return Packet2cf(vcombine_f32(vget_high_f32(a.v), vget_low_f32(a.v))); + return Packet2cf(vextq_f32(a.v, a.v, 2)); } template <> @@ -616,18 +614,14 @@ } template <> -EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from, - Index stride) { - Packet2d res = pset1<Packet2d>(0.0); - res = vsetq_lane_f64(std::real(from[0 * stride]), res, 0); - res = vsetq_lane_f64(std::imag(from[0 * stride]), res, 1); - return Packet1cd(res); +EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from, Index) { + return ploadu<Packet1cd>(from); } template <> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet1cd>(std::complex<double>* to, const Packet1cd& from, - Index stride) { - to[stride * 0] = std::complex<double>(vgetq_lane_f64(from.v, 0), vgetq_lane_f64(from.v, 1)); + Index) { + pstoreu(to, from); } template <>
diff --git a/Eigen/src/Core/arch/NEON/PacketMath.h b/Eigen/src/Core/arch/NEON/PacketMath.h index 678d53b..4465d7d 100644 --- a/Eigen/src/Core/arch/NEON/PacketMath.h +++ b/Eigen/src/Core/arch/NEON/PacketMath.h
@@ -1412,13 +1412,44 @@ return vaddq_u64(b, vqsubq_u64(a, b)); } +// GCC defines the ARM32 vcge/vcgt float intrinsics as generic vector comparisons and, unless +// -funsafe-math-optimizations is given, expands them one lane at a time through VFP, because Advanced SIMD +// flushes subnormal inputs. Emit the Advanced SIMD comparison directly, as Clang does. +#if EIGEN_COMP_GNUC_STRICT && !EIGEN_ARCH_ARM64 +EIGEN_STRONG_INLINE uint32x2_t neon_vcge_f32(float32x2_t a, float32x2_t b) { + uint32x2_t r; + __asm__("vcge.f32 %P0, %P1, %P2" : "=w"(r) : "w"(a), "w"(b)); + return r; +} +EIGEN_STRONG_INLINE uint32x4_t neon_vcgeq_f32(float32x4_t a, float32x4_t b) { + uint32x4_t r; + __asm__("vcge.f32 %q0, %q1, %q2" : "=w"(r) : "w"(a), "w"(b)); + return r; +} +EIGEN_STRONG_INLINE uint32x2_t neon_vcgt_f32(float32x2_t a, float32x2_t b) { + uint32x2_t r; + __asm__("vcgt.f32 %P0, %P1, %P2" : "=w"(r) : "w"(a), "w"(b)); + return r; +} +EIGEN_STRONG_INLINE uint32x4_t neon_vcgtq_f32(float32x4_t a, float32x4_t b) { + uint32x4_t r; + __asm__("vcgt.f32 %q0, %q1, %q2" : "=w"(r) : "w"(a), "w"(b)); + return r; +} +#else +EIGEN_STRONG_INLINE uint32x2_t neon_vcge_f32(float32x2_t a, float32x2_t b) { return vcge_f32(a, b); } +EIGEN_STRONG_INLINE uint32x4_t neon_vcgeq_f32(float32x4_t a, float32x4_t b) { return vcgeq_f32(a, b); } +EIGEN_STRONG_INLINE uint32x2_t neon_vcgt_f32(float32x2_t a, float32x2_t b) { return vcgt_f32(a, b); } +EIGEN_STRONG_INLINE uint32x4_t neon_vcgtq_f32(float32x4_t a, float32x4_t b) { return vcgtq_f32(a, b); } +#endif + template <> EIGEN_STRONG_INLINE Packet2f pcmp_le<Packet2f>(const Packet2f& a, const Packet2f& b) { - return vreinterpret_f32_u32(vcle_f32(a, b)); + return vreinterpret_f32_u32(neon_vcge_f32(b, a)); } template <> EIGEN_STRONG_INLINE Packet4f pcmp_le<Packet4f>(const Packet4f& a, const Packet4f& b) { - return vreinterpretq_f32_u32(vcleq_f32(a, b)); + return vreinterpretq_f32_u32(neon_vcgeq_f32(b, a)); } template <> EIGEN_STRONG_INLINE Packet4c pcmp_le<Packet4c>(const Packet4c& a, const Packet4c& b) { @@ -1481,11 +1512,11 @@ template <> EIGEN_STRONG_INLINE Packet2f pcmp_lt<Packet2f>(const Packet2f& a, const Packet2f& b) { - return vreinterpret_f32_u32(vclt_f32(a, b)); + return vreinterpret_f32_u32(neon_vcgt_f32(b, a)); } template <> EIGEN_STRONG_INLINE Packet4f pcmp_lt<Packet4f>(const Packet4f& a, const Packet4f& b) { - return vreinterpretq_f32_u32(vcltq_f32(a, b)); + return vreinterpretq_f32_u32(neon_vcgtq_f32(b, a)); } template <> EIGEN_STRONG_INLINE Packet4c pcmp_lt<Packet4c>(const Packet4c& a, const Packet4c& b) { @@ -1675,11 +1706,39 @@ template <> EIGEN_STRONG_INLINE Packet2f pcmp_lt_or_nan<Packet2f>(const Packet2f& a, const Packet2f& b) { - return vreinterpret_f32_u32(vmvn_u32(vcge_f32(a, b))); + return vreinterpret_f32_u32(vmvn_u32(neon_vcge_f32(a, b))); } template <> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(const Packet4f& a, const Packet4f& b) { - return vreinterpretq_f32_u32(vmvnq_u32(vcgeq_f32(a, b))); + return vreinterpretq_f32_u32(vmvnq_u32(neon_vcgeq_f32(a, b))); +} + +// Classify with absolute comparisons against infinity: |a| >= inf holds only for +-inf, inf > |a| only for finite +// values, and |a| <= inf fails only for NaN. Each is one instruction (FACGE/FACGT, VACGE/VACGT), is unaffected by +// flushing subnormal inputs, and avoids the self-comparison a != a, which Clang rewrites into two comparisons. +template <> +EIGEN_STRONG_INLINE Packet2f pisinf<Packet2f>(const Packet2f& a) { + return vreinterpret_f32_u32(vcage_f32(a, pinf<Packet2f>())); +} +template <> +EIGEN_STRONG_INLINE Packet4f pisinf<Packet4f>(const Packet4f& a) { + return vreinterpretq_f32_u32(vcageq_f32(a, pinf<Packet4f>())); +} +template <> +EIGEN_STRONG_INLINE Packet2f pisfinite<Packet2f>(const Packet2f& a) { + return vreinterpret_f32_u32(vcagt_f32(pinf<Packet2f>(), a)); +} +template <> +EIGEN_STRONG_INLINE Packet4f pisfinite<Packet4f>(const Packet4f& a) { + return vreinterpretq_f32_u32(vcagtq_f32(pinf<Packet4f>(), a)); +} +template <> +EIGEN_STRONG_INLINE Packet2f pisnan<Packet2f>(const Packet2f& a) { + return vreinterpret_f32_u32(vmvn_u32(vcage_f32(pinf<Packet2f>(), a))); +} +template <> +EIGEN_STRONG_INLINE Packet4f pisnan<Packet4f>(const Packet4f& a) { + return vreinterpretq_f32_u32(vmvnq_u32(vcageq_f32(pinf<Packet4f>(), a))); } // Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics @@ -2642,6 +2701,17 @@ } template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather<float, Packet4f>(const float* from, Index stride) { + if (stride == 2) { + // Overlap the loads so the second ends at the last gathered coefficient, from[6]. +#if EIGEN_ARCH_ARM64 + // lo = [0 1 2 3], hi = [3 4 5 6]. GCC turns the equivalent vuzp2q(vrev64q(lo), hi) into a slower TBL. + Packet4f lo = vld1q_f32(from); + Packet4f hi = vld1q_f32(from + 3); + return vcombine_f32(vget_low_f32(vuzp1q_f32(lo, lo)), vget_high_f32(vuzp2q_f32(hi, hi))); +#else + return vcombine_f32(vld2_f32(from).val[0], vld2_f32(from + 3).val[1]); +#endif + } Packet4f res = vld1q_dup_f32(from); res = vld1q_lane_f32(from + 1 * stride, res, 1); res = vld1q_lane_f32(from + 2 * stride, res, 2); @@ -4857,6 +4927,15 @@ return vcgt_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf)); } +// Compare encoded lanes: widening bf16 subnormals to float loses them under DAZ/FZ. +template <> +EIGEN_STRONG_INLINE Packet4bf psign<Packet4bf>(const Packet4bf& a) { + const uint16x4_t abs_mask = vdup_n_u16(0x7fff); + const uint16x4_t is_nan = vcgt_u16(vand_u16(a, abs_mask), vdup_n_u16(0x7f80)); + const uint16x4_t keep = vorr_u16(is_nan, vdup_n_u16(0x8000)); + return vand_u16(vbsl_u16(keep, a, vdup_n_u16(0x3f80)), vtst_u16(a, abs_mask)); +} + template <> EIGEN_STRONG_INLINE Packet4bf pisfinite<Packet4bf>(const Packet4bf& a) { constexpr uint16_t kInf = ((1 << 8) - 1) << 7; @@ -5300,6 +5379,22 @@ return vsqrtq_f64(_x); } +// Absolute comparisons as for Packet4f. +template <> +EIGEN_STRONG_INLINE Packet2d pisinf<Packet2d>(const Packet2d& a) { + return vreinterpretq_f64_u64(vcageq_f64(a, pinf<Packet2d>())); +} +template <> +EIGEN_STRONG_INLINE Packet2d pisfinite<Packet2d>(const Packet2d& a) { + return vreinterpretq_f64_u64(vcagtq_f64(pinf<Packet2d>(), a)); +} +// |bits| > inf bits as an unsigned comparison. !vcage(inf, a) is one instruction shorter inside a select, but Clang's +// inliner charges more for the intrinsic, and pexp_complex<Packet1cd> sits within a few units of its threshold. +template <> +EIGEN_STRONG_INLINE Packet2d pisnan<Packet2d>(const Packet2d& a) { + return vreinterpretq_f64_u64(vcgtq_u64(vreinterpretq_u64_f64(vabsq_f64(a)), vdupq_n_u64(0x7ff0000000000000ull))); +} + #endif // EIGEN_ARCH_ARM64 // Do we have fp16 and support Neon intrinsics? @@ -5813,6 +5908,32 @@ return vreinterpret_f16_u16(vmvn_u16(vcge_f16(a, b))); } +// Absolute comparisons as for Packet4f. +template <> +EIGEN_STRONG_INLINE Packet4hf pisinf<Packet4hf>(const Packet4hf& a) { + return vreinterpret_f16_u16(vcage_f16(a, pinf<Packet4hf>())); +} +template <> +EIGEN_STRONG_INLINE Packet8hf pisinf<Packet8hf>(const Packet8hf& a) { + return vreinterpretq_f16_u16(vcageq_f16(a, pinf<Packet8hf>())); +} +template <> +EIGEN_STRONG_INLINE Packet4hf pisfinite<Packet4hf>(const Packet4hf& a) { + return vreinterpret_f16_u16(vcagt_f16(pinf<Packet4hf>(), a)); +} +template <> +EIGEN_STRONG_INLINE Packet8hf pisfinite<Packet8hf>(const Packet8hf& a) { + return vreinterpretq_f16_u16(vcagtq_f16(pinf<Packet8hf>(), a)); +} +template <> +EIGEN_STRONG_INLINE Packet4hf pisnan<Packet4hf>(const Packet4hf& a) { + return vreinterpret_f16_u16(vmvn_u16(vcage_f16(pinf<Packet4hf>(), a))); +} +template <> +EIGEN_STRONG_INLINE Packet8hf pisnan<Packet8hf>(const Packet8hf& a) { + return vreinterpretq_f16_u16(vmvnq_u16(vcageq_f16(pinf<Packet8hf>(), a))); +} + template <> EIGEN_STRONG_INLINE Packet8hf print<Packet8hf>(const Packet8hf& a) { return vrndnq_f16(a); @@ -6179,6 +6300,41 @@ #endif // end EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC +#if !EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC +// Without FP16 arithmetic, half comparisons convert lane by lane. Classify on the raw bits instead, as for bfloat16: +// |a| ==, >, < 0x7c00. +EIGEN_STRONG_INLINE uint16x4_t neon_half_abs_bits(const Packet4hf& a) { + return vbic_u16(vreinterpret_u16_f16(a), vdup_n_u16(0x8000)); +} +EIGEN_STRONG_INLINE uint16x8_t neon_half_abs_bits(const Packet8hf& a) { + return vbicq_u16(vreinterpretq_u16_f16(a), vdupq_n_u16(0x8000)); +} +template <> +EIGEN_STRONG_INLINE Packet4hf pisinf<Packet4hf>(const Packet4hf& a) { + return vreinterpret_f16_u16(vceq_u16(neon_half_abs_bits(a), vdup_n_u16(0x7c00))); +} +template <> +EIGEN_STRONG_INLINE Packet8hf pisinf<Packet8hf>(const Packet8hf& a) { + return vreinterpretq_f16_u16(vceqq_u16(neon_half_abs_bits(a), vdupq_n_u16(0x7c00))); +} +template <> +EIGEN_STRONG_INLINE Packet4hf pisnan<Packet4hf>(const Packet4hf& a) { + return vreinterpret_f16_u16(vcgt_u16(neon_half_abs_bits(a), vdup_n_u16(0x7c00))); +} +template <> +EIGEN_STRONG_INLINE Packet8hf pisnan<Packet8hf>(const Packet8hf& a) { + return vreinterpretq_f16_u16(vcgtq_u16(neon_half_abs_bits(a), vdupq_n_u16(0x7c00))); +} +template <> +EIGEN_STRONG_INLINE Packet4hf pisfinite<Packet4hf>(const Packet4hf& a) { + return vreinterpret_f16_u16(vclt_u16(neon_half_abs_bits(a), vdup_n_u16(0x7c00))); +} +template <> +EIGEN_STRONG_INLINE Packet8hf pisfinite<Packet8hf>(const Packet8hf& a) { + return vreinterpretq_f16_u16(vcltq_u16(neon_half_abs_bits(a), vdupq_n_u16(0x7c00))); +} +#endif + #define EIGEN_MAKE_HALF_NEG_FMA(name, base, packet) \ template <> \ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE packet name(const packet& a, const packet& b, const packet& c) { \
diff --git a/Eigen/src/Core/arch/RVV10/PacketMathDecl.h b/Eigen/src/Core/arch/RVV10/PacketMathDecl.h index e17aa76..e51675f 100644 --- a/Eigen/src/Core/arch/RVV10/PacketMathDecl.h +++ b/Eigen/src/Core/arch/RVV10/PacketMathDecl.h
@@ -287,11 +287,24 @@ HasSin = EIGEN_FAST_MATH, HasCos = EIGEN_FAST_MATH, HasTan = EIGEN_FAST_MATH, + HasSinh = 1, + HasCosh = 1, + HasASinh = 1, + HasACosh = 1, + HasATan = 1, + HasATanh = 1, HasLog = 1, HasExp = 1, + HasLog10 = 1, + HasLog1p = 1, + HasExpm1 = 1, + HasPow = 1, HasSqrt = 1, + HasCbrt = 1, + HasRsqrt = 1, HasTanh = EIGEN_FAST_MATH, - HasErf = EIGEN_FAST_MATH + HasErf = EIGEN_FAST_MATH, + HasErfc = EIGEN_FAST_MATH }; };
diff --git a/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h b/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h index 0e4cd83..a5a5815 100644 --- a/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h +++ b/Eigen/src/Core/arch/SME/GeneralBlockPanelKernel.h
@@ -56,231 +56,6 @@ // to a hand-scheduled multi-vector-load loop (see sme_process). // --------------------------------------------------------------------------- -// The per-element-width half of the ACLE surface: everything the kernel needs -// that is selected by the element type alone rather than by an argument. -// Operations that can be overloaded on their arguments are free functions -// below, and the ones taking a ZA tile number take it as a template parameter -// because the underlying instructions encode it as an immediate. -// -// `whilelt` takes int64_t in both specializations rather than the caller's own -// integer type: svwhilelt_b* is overloaded on the four fixed-width types only, -// so an argument of a distinct type of the same width -- `Index` wherever -// int64_t is `long long` -- matches none of them exactly. Signed, because -// every bound the packers pass is a non-negative Index. -template <typename Scalar> -struct sme_traits; - -// ZA tile count at a given element width: 4 ZA.S tiles, 8 ZA.D tiles. This is -// architectural rather than feature-dependent, so it stays outside sme_traits, -// which has no double specialization without FEAT_SME_F64F64. -template <typename RealScalar> -struct sme_tile_count; -template <> -struct sme_tile_count<float> { - static constexpr int value = 4; -}; -template <> -struct sme_tile_count<double> { - static constexpr int value = 8; -}; - -template <> -struct sme_traits<float> { - using Vec = svfloat32_t; - using Vec2 = svfloat32x2_t; - using Vec4 = svfloat32x4_t; - static EIGEN_ALWAYS_INLINE int svl() __arm_streaming_compatible { return static_cast<int>(svcntsw()); } - static EIGEN_ALWAYS_INLINE svbool_t whilelt(int64_t begin, int64_t end) __arm_streaming { - return svwhilelt_b32(begin, end); - } - static EIGEN_ALWAYS_INLINE svbool_t ptrue() __arm_streaming { return svptrue_b32(); } - static EIGEN_ALWAYS_INLINE svcount_t ptrue_c() __arm_streaming { return svptrue_c32(); } - static EIGEN_ALWAYS_INLINE Vec dup(float x) __arm_streaming { return svdup_f32(x); } -}; - -#ifdef EIGEN_VECTORIZE_SME_F64F64 -template <> -struct sme_traits<double> { - using Vec = svfloat64_t; - using Vec2 = svfloat64x2_t; - using Vec4 = svfloat64x4_t; - static EIGEN_ALWAYS_INLINE int svl() __arm_streaming_compatible { return static_cast<int>(svcntsd()); } - static EIGEN_ALWAYS_INLINE svbool_t whilelt(int64_t begin, int64_t end) __arm_streaming { - return svwhilelt_b64(begin, end); - } - static EIGEN_ALWAYS_INLINE svbool_t ptrue() __arm_streaming { return svptrue_b64(); } - static EIGEN_ALWAYS_INLINE svcount_t ptrue_c() __arm_streaming { return svptrue_c64(); } - static EIGEN_ALWAYS_INLINE Vec dup(double x) __arm_streaming { return svdup_f64(x); } -}; -#endif - -// Contiguous predicated load/store, fused multiply-add and multiply. -static EIGEN_ALWAYS_INLINE svfloat32_t sme_ld1(svbool_t pg, const float* p) __arm_streaming { return svld1_f32(pg, p); } -static EIGEN_ALWAYS_INLINE void sme_st1(svbool_t pg, float* p, svfloat32_t v) __arm_streaming { svst1_f32(pg, p, v); } -static EIGEN_ALWAYS_INLINE svfloat32x2_t sme_ld1_x2(svcount_t pn, const float* p) __arm_streaming { - return svld1_f32_x2(pn, p); -} -static EIGEN_ALWAYS_INLINE svfloat32x4_t sme_ld1_x4(svcount_t pn, const float* p) __arm_streaming { - return svld1_f32_x4(pn, p); -} -static EIGEN_ALWAYS_INLINE svfloat32_t sme_mla(svbool_t pg, svfloat32_t acc, svfloat32_t a, - svfloat32_t b) __arm_streaming { - return svmla_f32_x(pg, acc, a, b); -} -static EIGEN_ALWAYS_INLINE svfloat32_t sme_mul(svbool_t pg, svfloat32_t a, svfloat32_t b) __arm_streaming { - return svmul_f32_x(pg, a, b); -} -template <int Lane> -static EIGEN_ALWAYS_INLINE svfloat32_t sme_get(svfloat32x2_t v) __arm_streaming { - return svget2_f32(v, Lane); -} -template <int Lane> -static EIGEN_ALWAYS_INLINE svfloat32_t sme_get(svfloat32x4_t v) __arm_streaming { - return svget4_f32(v, Lane); -} - -// ZA tile access. The tile number is an instruction immediate, hence a template -// parameter; the slice number is a register operand and stays a value. -template <int Tile> -static EIGEN_ALWAYS_INLINE void sme_ld1_hor_za(uint32_t slice, svbool_t pg, - const float* p) __arm_streaming __arm_inout("za") { - svld1_hor_za32(Tile, slice, pg, p); -} -template <int Tile> -static EIGEN_ALWAYS_INLINE svfloat32_t sme_read_hor_za(svfloat32_t zero, svbool_t pg, - uint32_t slice) __arm_streaming __arm_inout("za") { - return svread_hor_za32_f32_m(zero, pg, Tile, slice); -} -template <int Tile> -static EIGEN_ALWAYS_INLINE svfloat32_t sme_read_ver_za(svfloat32_t zero, svbool_t pg, - uint32_t slice) __arm_streaming __arm_inout("za") { - return svread_ver_za32_f32_m(zero, pg, Tile, slice); -} -template <int Tile> -static EIGEN_ALWAYS_INLINE void sme_mopa(svbool_t pm, svbool_t pn, svfloat32_t a, - svfloat32_t b) __arm_streaming __arm_inout("za") { - svmopa_za32_f32_m(Tile, pm, pn, a, b); -} -template <int Tile> -static EIGEN_ALWAYS_INLINE void sme_mops(svbool_t pm, svbool_t pn, svfloat32_t a, - svfloat32_t b) __arm_streaming __arm_inout("za") { - svmops_za32_f32_m(Tile, pm, pn, a, b); -} - -// Fused multiply-subtract, negation, and the de-/interleaving permutes the -// complex kernel needs on top of the real one. -static EIGEN_ALWAYS_INLINE svfloat32_t sme_mls(svbool_t pg, svfloat32_t acc, svfloat32_t a, - svfloat32_t b) __arm_streaming { - return svmls_f32_x(pg, acc, a, b); -} -static EIGEN_ALWAYS_INLINE svfloat32_t sme_add(svbool_t pg, svfloat32_t a, svfloat32_t b) __arm_streaming { - return svadd_f32_x(pg, a, b); -} -static EIGEN_ALWAYS_INLINE svfloat32_t sme_neg(svbool_t pg, svfloat32_t v) __arm_streaming { - return svneg_f32_x(pg, v); -} -static EIGEN_ALWAYS_INLINE svfloat32_t sme_uzp1(svfloat32_t a, svfloat32_t b) __arm_streaming { - return svuzp1_f32(a, b); -} -static EIGEN_ALWAYS_INLINE svfloat32_t sme_uzp2(svfloat32_t a, svfloat32_t b) __arm_streaming { - return svuzp2_f32(a, b); -} -static EIGEN_ALWAYS_INLINE svfloat32_t sme_zip1(svfloat32_t a, svfloat32_t b) __arm_streaming { - return svzip1_f32(a, b); -} -static EIGEN_ALWAYS_INLINE svfloat32_t sme_zip2(svfloat32_t a, svfloat32_t b) __arm_streaming { - return svzip2_f32(a, b); -} - -#ifdef EIGEN_VECTORIZE_SME_F64F64 -static EIGEN_ALWAYS_INLINE svfloat64_t sme_ld1(svbool_t pg, const double* p) __arm_streaming { - return svld1_f64(pg, p); -} -static EIGEN_ALWAYS_INLINE void sme_st1(svbool_t pg, double* p, svfloat64_t v) __arm_streaming { svst1_f64(pg, p, v); } -static EIGEN_ALWAYS_INLINE svfloat64x2_t sme_ld1_x2(svcount_t pn, const double* p) __arm_streaming { - return svld1_f64_x2(pn, p); -} -static EIGEN_ALWAYS_INLINE svfloat64x4_t sme_ld1_x4(svcount_t pn, const double* p) __arm_streaming { - return svld1_f64_x4(pn, p); -} -static EIGEN_ALWAYS_INLINE svfloat64_t sme_mla(svbool_t pg, svfloat64_t acc, svfloat64_t a, - svfloat64_t b) __arm_streaming { - return svmla_f64_x(pg, acc, a, b); -} -static EIGEN_ALWAYS_INLINE svfloat64_t sme_mul(svbool_t pg, svfloat64_t a, svfloat64_t b) __arm_streaming { - return svmul_f64_x(pg, a, b); -} -template <int Lane> -static EIGEN_ALWAYS_INLINE svfloat64_t sme_get(svfloat64x2_t v) __arm_streaming { - return svget2_f64(v, Lane); -} -template <int Lane> -static EIGEN_ALWAYS_INLINE svfloat64_t sme_get(svfloat64x4_t v) __arm_streaming { - return svget4_f64(v, Lane); -} - -template <int Tile> -static EIGEN_ALWAYS_INLINE void sme_ld1_hor_za(uint32_t slice, svbool_t pg, - const double* p) __arm_streaming __arm_inout("za") { - svld1_hor_za64(Tile, slice, pg, p); -} -template <int Tile> -static EIGEN_ALWAYS_INLINE svfloat64_t sme_read_hor_za(svfloat64_t zero, svbool_t pg, - uint32_t slice) __arm_streaming __arm_inout("za") { - return svread_hor_za64_f64_m(zero, pg, Tile, slice); -} -template <int Tile> -static EIGEN_ALWAYS_INLINE svfloat64_t sme_read_ver_za(svfloat64_t zero, svbool_t pg, - uint32_t slice) __arm_streaming __arm_inout("za") { - return svread_ver_za64_f64_m(zero, pg, Tile, slice); -} -template <int Tile> -static EIGEN_ALWAYS_INLINE void sme_mopa(svbool_t pm, svbool_t pn, svfloat64_t a, - svfloat64_t b) __arm_streaming __arm_inout("za") { - svmopa_za64_f64_m(Tile, pm, pn, a, b); -} -template <int Tile> -static EIGEN_ALWAYS_INLINE void sme_mops(svbool_t pm, svbool_t pn, svfloat64_t a, - svfloat64_t b) __arm_streaming __arm_inout("za") { - svmops_za64_f64_m(Tile, pm, pn, a, b); -} - -static EIGEN_ALWAYS_INLINE svfloat64_t sme_mls(svbool_t pg, svfloat64_t acc, svfloat64_t a, - svfloat64_t b) __arm_streaming { - return svmls_f64_x(pg, acc, a, b); -} -static EIGEN_ALWAYS_INLINE svfloat64_t sme_add(svbool_t pg, svfloat64_t a, svfloat64_t b) __arm_streaming { - return svadd_f64_x(pg, a, b); -} -static EIGEN_ALWAYS_INLINE svfloat64_t sme_neg(svbool_t pg, svfloat64_t v) __arm_streaming { - return svneg_f64_x(pg, v); -} -static EIGEN_ALWAYS_INLINE svfloat64_t sme_uzp1(svfloat64_t a, svfloat64_t b) __arm_streaming { - return svuzp1_f64(a, b); -} -static EIGEN_ALWAYS_INLINE svfloat64_t sme_uzp2(svfloat64_t a, svfloat64_t b) __arm_streaming { - return svuzp2_f64(a, b); -} -static EIGEN_ALWAYS_INLINE svfloat64_t sme_zip1(svfloat64_t a, svfloat64_t b) __arm_streaming { - return svzip1_f64(a, b); -} -static EIGEN_ALWAYS_INLINE svfloat64_t sme_zip2(svfloat64_t a, svfloat64_t b) __arm_streaming { - return svzip2_f64(a, b); -} -#endif // EIGEN_VECTORIZE_SME_F64F64 - -// Outer-product accumulate with a compile-time sign: the complex kernel's four -// real products differ only in whether they add or subtract into the tile. -template <int Tile, bool Subtract, typename Vec> -static EIGEN_ALWAYS_INLINE void sme_mopa_signed(svbool_t pm, svbool_t pn, Vec a, - Vec b) __arm_streaming __arm_inout("za") { - EIGEN_IF_CONSTEXPR (Subtract) { - sme_mops<Tile>(pm, pn, a, b); - } else { - sme_mopa<Tile>(pm, pn, a, b); - } -} - // The streaming vector width the block sizes are chosen at: 512 bits, where a // vector holds 64 / sizeof(element) elements. Other SVLs tile the block at // runtime. If a future SVL ever justifies a larger block, this is the only @@ -322,22 +97,6 @@ static constexpr int kSmeNrCD = sme_block<std::complex<double>>::nr; #endif -// min() usable from streaming functions (numext::mini lacks the -// __arm_streaming_compatible attribute). -template <typename T> -static EIGEN_ALWAYS_INLINE T sme_min(T a, T b) __arm_streaming_compatible { - return a < b ? a : b; -} - -// Offset a pointer by n elements without forming the pointer value: an access -// whose predicate is empty makes no memory reference, but computing an address -// more than one past the end of the object is undefined regardless, so the -// second-vector accesses below reach their address through uintptr_t. -template <typename T> -static EIGEN_ALWAYS_INLINE T* sme_offset(T* p, Index n) __arm_streaming_compatible { - return reinterpret_cast<T*>(uintptr_t(p) + ptrdiff_t(n) * sizeof(T)); -} - // --------------------------------------------------------------------------- // Packed panel layout and the primitives that produce it. // @@ -366,15 +125,77 @@ template <bool Conjugate, typename Scalar, typename Index> static EIGEN_ALWAYS_INLINE void sve_copy_panel_range(Scalar* EIGEN_RESTRICT dst, const Scalar* EIGEN_RESTRICT src, Index src_stride, Index k0, Index k1, int width) __arm_streaming { - const int svl = sme_traits<Scalar>::svl(); + using Traits = sme_packet_traits<Scalar>; + const int svl = Traits::size(); + if (width == 2 * svl) { + // Full panel: one two-vector load and store per depth step, four steps in flight. + const svcount_t pn = Traits::ptrue_c(); + Index k = k0; + for (; k + 4 <= k1; k += 4) { + const auto v0 = ploadu_x2(pn, &src[k * src_stride]); + const auto v1 = ploadu_x2(pn, &src[(k + 1) * src_stride]); + const auto v2 = ploadu_x2(pn, &src[(k + 2) * src_stride]); + const auto v3 = ploadu_x2(pn, &src[(k + 3) * src_stride]); + pstoreu_x2(pn, &dst[k * width], v0); + pstoreu_x2(pn, &dst[(k + 1) * width], v1); + pstoreu_x2(pn, &dst[(k + 2) * width], v2); + pstoreu_x2(pn, &dst[(k + 3) * width], v3); + } + for (; k < k1; ++k) pstoreu_x2(pn, &dst[k * width], ploadu_x2(pn, &src[k * src_stride])); + return; + } for (int off = 0; off < width; off += svl) { - const svbool_t pred = sme_traits<Scalar>::whilelt(off, width); + const svbool_t pred = sme_packet_traits<Scalar>::whilelt(off, width); for (Index k = k0; k < k1; ++k) { - sme_st1(pred, &dst[k * width + off], sme_ld1(pred, &src[k * src_stride + off])); + pstoreu(pred, &dst[k * width + off], ploadu(pred, &src[k * src_stride + off])); } } } +// Four full 2*svl-wide panels from one 8*svl-wide source strip (panel p at dst + p * panel_stride): two four-vector +// loads per depth step, and the core prefetches the strip into L2 four steps ahead. +template <typename Scalar, typename Index> +static EIGEN_ALWAYS_INLINE void sve_copy_panel_quad(Scalar* EIGEN_RESTRICT dst, Index panel_stride, + const Scalar* EIGEN_RESTRICT src, Index src_stride, + Index depth) __arm_streaming { + using Traits = sme_packet_traits<Scalar>; + const int w = 2 * Traits::size(); + const svcount_t pn = Traits::ptrue_c(); + const int strip_bytes = 4 * w * int(sizeof(Scalar)); + for (Index k = 0; k < depth; ++k) { + const Scalar* c = src + k * src_stride; + if (k + 4 < depth) + for (int l = 0; l < strip_bytes; l += 128) + __builtin_prefetch(reinterpret_cast<const char*>(c + 4 * src_stride) + l, 0, 2); + const auto a = ploadu_x4(pn, c), b = ploadu_x4(pn, c + 2 * w); + pstoreu_x2(pn, dst + k * w, pcreate(pget<0>(a), pget<1>(a))); + pstoreu_x2(pn, dst + panel_stride + k * w, pcreate(pget<2>(a), pget<3>(a))); + pstoreu_x2(pn, dst + 2 * panel_stride + k * w, pcreate(pget<0>(b), pget<1>(b))); + pstoreu_x2(pn, dst + 3 * panel_stride + k * w, pcreate(pget<2>(b), pget<3>(b))); + } +} + +// Packs the leading groups of four full MR-wide panels with sve_copy_panel_quad; returns the rows it covered. +// Complex panels keep the per-panel path (their packed layout splits real and imaginary halves). +template <bool PanelMode, typename Scalar, typename Index> +static EIGEN_ALWAYS_INLINE Index sme_pack_quad_panels(Scalar* dst_base, const Scalar* EIGEN_RESTRICT src, + Index src_stride, Index depth, Index rows, Index mr, + Index dst_stride, Index dst_offset) __arm_streaming { + if (mr != 2 * sme_packet_traits<Scalar>::size()) return 0; + const Index panel_stride = PanelMode ? mr * dst_stride : mr * depth; + Index i = 0; + for (; i + 4 * mr <= rows; i += 4 * mr) { + Scalar* dst_panel = PanelMode ? dst_base + i * dst_stride + dst_offset * mr : dst_base + i * depth; + sve_copy_panel_quad(dst_panel, panel_stride, src + i, src_stride, depth); + } + return i; +} +template <bool PanelMode, typename RealScalar, typename Index> +static EIGEN_ALWAYS_INLINE Index sme_pack_quad_panels(std::complex<RealScalar>*, const std::complex<RealScalar>*, Index, + Index, Index, Index, Index, Index) __arm_streaming { + return 0; +} + // Complex overload: UZP1/UZP2 split the source's interleaved pairs into the two // halves of the packed depth step. A chunk of w complex elements spans 2*w // interleaved reals, hence a pair of source predicates; the upper one is empty @@ -383,11 +204,11 @@ static EIGEN_ALWAYS_INLINE void sve_copy_panel_range(std::complex<RealScalar>* EIGEN_RESTRICT dst, const std::complex<RealScalar>* EIGEN_RESTRICT src, Index src_stride, Index k0, Index k1, int width) __arm_streaming { - using Traits = sme_traits<RealScalar>; - using Vec = typename Traits::Vec; + using Traits = sme_packet_traits<RealScalar>; + using Vec = typename Traits::type; RealScalar* EIGEN_RESTRICT rdst = reinterpret_cast<RealScalar*>(dst); const RealScalar* EIGEN_RESTRICT rsrc = reinterpret_cast<const RealScalar*>(src); - const int svl = Traits::svl(); + const int svl = Traits::size(); const Index step = Index(2 * width); for (int off = 0; off < width; off += svl) { const int w = width - off; @@ -397,17 +218,17 @@ const svbool_t pg_hi = Traits::whilelt(svl, lanes); for (Index k = k0; k < k1; ++k) { const RealScalar* p = rsrc + Index(2) * (k * src_stride + Index(off)); - const Vec v_lo = sme_ld1(pg_lo, p); + const Vec v_lo = ploadu(pg_lo, p); // pg_hi is all-false when 2*w fits in one vector, and an inactive lane // makes no memory access -- but p + svl may still be past the source, so // the address is formed through sme_offset. - const Vec v_hi = sme_ld1(pg_hi, sme_offset(p, svl)); - Vec im = sme_uzp2(v_lo, v_hi); + const Vec v_hi = ploadu(pg_hi, sme_offset(p, svl)); + Vec im = puzp2(v_lo, v_hi); EIGEN_IF_CONSTEXPR (Conjugate) { - im = sme_neg(pg_w, im); + im = pnegate(pg_w, im); } - sme_st1(pg_w, &rdst[k * step + Index(off)], sme_uzp1(v_lo, v_hi)); - sme_st1(pg_w, &rdst[k * step + Index(width + off)], im); + pstoreu(pg_w, &rdst[k * step + Index(off)], puzp1(v_lo, v_hi)); + pstoreu(pg_w, &rdst[k * step + Index(width + off)], im); } } } @@ -420,6 +241,141 @@ sve_copy_panel_range<Conjugate>(dst, src, src_stride, Index(0), depth, width); } +// Full 2*svl-wide panel through all four tiles, four rows or depth steps per ZA move, source prefetched into L2 +// two steps ahead; returns the first depth index of the tail (< 2*svl) it leaves to the caller. +template <typename RealScalar, typename Index> +static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_pair(RealScalar* EIGEN_RESTRICT dst, + const RealScalar* EIGEN_RESTRICT src, Index src_stride, + Index k0, Index k1) __arm_streaming __arm_inout("za") { + using Traits = sme_packet_traits<RealScalar>; + const int svl = Traits::size(), w = 2 * svl; + const svcount_t pn = Traits::ptrue_c(); + Index k = k0; + for (; k + w <= k1; k += w) { + const bool pf = k + 2 * w <= k1; + for (int r = 0; r < svl; r += 4) { + const RealScalar* p = src + Index(r) * src_stride + k; + const RealScalar* q = p + Index(svl) * src_stride; + if (pf) + for (int u = 0; u < 4; ++u) { + __builtin_prefetch(p + u * src_stride + 2 * w, 0, 2); + __builtin_prefetch(q + u * src_stride + 2 * w, 0, 2); + } + const auto a0 = ploadu_x2(pn, p), a1 = ploadu_x2(pn, p + src_stride), a2 = ploadu_x2(pn, p + 2 * src_stride), + a3 = ploadu_x2(pn, p + 3 * src_stride); + sme_write_hor_za_vg4<0>(uint32_t(r), pget<0>(a0), pget<0>(a1), pget<0>(a2), pget<0>(a3)); + sme_write_hor_za_vg4<1>(uint32_t(r), pget<1>(a0), pget<1>(a1), pget<1>(a2), pget<1>(a3)); + const auto b0 = ploadu_x2(pn, q), b1 = ploadu_x2(pn, q + src_stride), b2 = ploadu_x2(pn, q + 2 * src_stride), + b3 = ploadu_x2(pn, q + 3 * src_stride); + sme_write_hor_za_vg4<2>(uint32_t(r), pget<0>(b0), pget<0>(b1), pget<0>(b2), pget<0>(b3)); + sme_write_hor_za_vg4<3>(uint32_t(r), pget<1>(b0), pget<1>(b1), pget<1>(b2), pget<1>(b3)); + } + for (int c = 0; c < svl; c += 4) { + const auto t0 = sme_read_ver_za_vg4<0, RealScalar>(uint32_t(c)), + t2 = sme_read_ver_za_vg4<2, RealScalar>(uint32_t(c)); + const auto t1 = sme_read_ver_za_vg4<1, RealScalar>(uint32_t(c)), + t3 = sme_read_ver_za_vg4<3, RealScalar>(uint32_t(c)); + RealScalar* d0 = dst + (k + c) * w; + RealScalar* d1 = d0 + Index(svl) * w; + pstoreu_x2(pn, d0, pcreate(pget<0>(t0), pget<0>(t2))); + pstoreu_x2(pn, d0 + w, pcreate(pget<1>(t0), pget<1>(t2))); + pstoreu_x2(pn, d0 + 2 * w, pcreate(pget<2>(t0), pget<2>(t2))); + pstoreu_x2(pn, d0 + 3 * w, pcreate(pget<3>(t0), pget<3>(t2))); + pstoreu_x2(pn, d1, pcreate(pget<0>(t1), pget<0>(t3))); + pstoreu_x2(pn, d1 + w, pcreate(pget<1>(t1), pget<1>(t3))); + pstoreu_x2(pn, d1 + 2 * w, pcreate(pget<2>(t1), pget<2>(t3))); + pstoreu_x2(pn, d1 + 3 * w, pcreate(pget<3>(t1), pget<3>(t3))); + } + } + return k; +} + +// A row of a partial panel, or zeros past its last row (those slices are never stored). +template <typename RealScalar> +static EIGEN_ALWAYS_INLINE typename sme_packet_traits<RealScalar>::type_x2 sme_row_or_zero( + const RealScalar* p, bool valid, svcount_t pn, typename sme_packet_traits<RealScalar>::type zero) __arm_streaming { + return valid ? ploadu_x2(pn, p) : pcreate(zero, zero); +} + +// Partial panel (width < 2*svl) through the tiles as sme_transpose_pack_pair does, rows past the width as zeros and +// predicated stores of `width` scalars per depth step; returns the first depth index of the tail it leaves. +template <typename RealScalar, typename Index> +static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_partial(RealScalar* EIGEN_RESTRICT dst, + const RealScalar* EIGEN_RESTRICT src, Index src_stride, + Index k0, Index k1, + int width) __arm_streaming __arm_inout("za") { + using Traits = sme_packet_traits<RealScalar>; + using Vec = typename Traits::type; + const int svl = Traits::size(), w2 = 2 * svl; + const svcount_t pn = Traits::ptrue_c(); + const Vec zero = pset1<Vec>(RealScalar(0)); + const int rows_lo = sme_min(width, svl), rows_hi = width - rows_lo; + const svbool_t pg_lo = Traits::whilelt(0, rows_lo), pg_hi = Traits::whilelt(0, rows_hi); + const svbool_t pg_two = Traits::whilelt(0, 2 * width); + Index k = k0; + for (; k + w2 <= k1; k += w2) { + for (int r = 0; r < rows_lo; r += 4) { + const RealScalar* q = src + Index(r) * src_stride + k; + const auto a0 = sme_row_or_zero<RealScalar>(q, r < rows_lo, pn, zero); + const auto a1 = sme_row_or_zero<RealScalar>(q + src_stride, r + 1 < rows_lo, pn, zero); + const auto a2 = sme_row_or_zero<RealScalar>(q + 2 * src_stride, r + 2 < rows_lo, pn, zero); + const auto a3 = sme_row_or_zero<RealScalar>(q + 3 * src_stride, r + 3 < rows_lo, pn, zero); + sme_write_hor_za_vg4<0>(uint32_t(r), pget<0>(a0), pget<0>(a1), pget<0>(a2), pget<0>(a3)); + sme_write_hor_za_vg4<1>(uint32_t(r), pget<1>(a0), pget<1>(a1), pget<1>(a2), pget<1>(a3)); + } + for (int r = 0; r < rows_hi; r += 4) { + const RealScalar* q = src + Index(svl + r) * src_stride + k; + const auto a0 = sme_row_or_zero<RealScalar>(q, r < rows_hi, pn, zero); + const auto a1 = sme_row_or_zero<RealScalar>(q + src_stride, r + 1 < rows_hi, pn, zero); + const auto a2 = sme_row_or_zero<RealScalar>(q + 2 * src_stride, r + 2 < rows_hi, pn, zero); + const auto a3 = sme_row_or_zero<RealScalar>(q + 3 * src_stride, r + 3 < rows_hi, pn, zero); + sme_write_hor_za_vg4<2>(uint32_t(r), pget<0>(a0), pget<0>(a1), pget<0>(a2), pget<0>(a3)); + sme_write_hor_za_vg4<3>(uint32_t(r), pget<1>(a0), pget<1>(a1), pget<1>(a2), pget<1>(a3)); + } + if (2 * width <= svl) { + // Two depth steps per store: half as many stores, and full lines at width svl/2. + for (int c = 0; c < svl; c += 4) { + const auto t0 = sme_read_ver_za_vg4<0, RealScalar>(uint32_t(c)), + t1 = sme_read_ver_za_vg4<1, RealScalar>(uint32_t(c)); + RealScalar* d0 = dst + (k + c) * width; + RealScalar* d1 = d0 + Index(svl) * width; + pstoreu(pg_two, d0, psplice(pg_lo, pget<0>(t0), pget<1>(t0))); + pstoreu(pg_two, d0 + 2 * width, psplice(pg_lo, pget<2>(t0), pget<3>(t0))); + pstoreu(pg_two, d1, psplice(pg_lo, pget<0>(t1), pget<1>(t1))); + pstoreu(pg_two, d1 + 2 * width, psplice(pg_lo, pget<2>(t1), pget<3>(t1))); + } + continue; + } + for (int c = 0; c < svl; c += 4) { + const auto t0 = sme_read_ver_za_vg4<0, RealScalar>(uint32_t(c)), + t1 = sme_read_ver_za_vg4<1, RealScalar>(uint32_t(c)); + RealScalar* d0 = dst + (k + c) * width; + RealScalar* d1 = d0 + Index(svl) * width; + pstoreu(pg_lo, d0, pget<0>(t0)); + pstoreu(pg_lo, d0 + width, pget<1>(t0)); + pstoreu(pg_lo, d0 + 2 * width, pget<2>(t0)); + pstoreu(pg_lo, d0 + 3 * width, pget<3>(t0)); + pstoreu(pg_lo, d1, pget<0>(t1)); + pstoreu(pg_lo, d1 + width, pget<1>(t1)); + pstoreu(pg_lo, d1 + 2 * width, pget<2>(t1)); + pstoreu(pg_lo, d1 + 3 * width, pget<3>(t1)); + if (rows_hi > 0) { + const auto t2 = sme_read_ver_za_vg4<2, RealScalar>(uint32_t(c)), + t3 = sme_read_ver_za_vg4<3, RealScalar>(uint32_t(c)); + pstoreu(pg_hi, d0 + svl, pget<0>(t2)); + pstoreu(pg_hi, d0 + width + svl, pget<1>(t2)); + pstoreu(pg_hi, d0 + 2 * width + svl, pget<2>(t2)); + pstoreu(pg_hi, d0 + 3 * width + svl, pget<3>(t2)); + pstoreu(pg_hi, d1 + svl, pget<0>(t3)); + pstoreu(pg_hi, d1 + width + svl, pget<1>(t3)); + pstoreu(pg_hi, d1 + 2 * width + svl, pget<2>(t3)); + pstoreu(pg_hi, d1 + 3 * width + svl, pget<3>(t3)); + } + } + } + return k; +} + // Transpose-pack `width` source rows into depth-major packed output using ZA's // 2D store as a free transpose, for the depth sub-range [k0, k1): a svl x svl // block of source (svl rows x svl depth) is loaded as horizontal ZA slices, @@ -445,13 +401,22 @@ const RealScalar* EIGEN_RESTRICT src, Index src_stride, Index k0, Index k1, int width) __arm_streaming __arm_inout("za") { - using Traits = sme_traits<RealScalar>; - using Vec = typename Traits::Vec; - const Vec zero = Traits::dup(RealScalar(0)); + using Traits = sme_packet_traits<RealScalar>; + using Vec = typename Traits::type; + const Vec zero = pset1<Vec>(RealScalar(0)); const svbool_t pg_all = Traits::ptrue(); - const int svl = Traits::svl(); + const int svl = Traits::size(); - for (Index k = k0; k < k1; k += svl) { + Index k = k0; + EIGEN_IF_CONSTEXPR (!NegateOddRows) { + // Short ranges keep the single-tile path: the four-slice moves only pay off over several fills, and need a tile + // of at least four slices. + if (svl >= 4 && width == 2 * svl && k1 - k0 >= Index(8 * svl)) + k = sme_transpose_pack_pair(dst, src, src_stride, k0, k1); + else if (svl >= 4 && width < 2 * svl && k1 - k0 >= Index(8 * svl)) + k = sme_transpose_pack_partial(dst, src, src_stride, k0, k1, width); + } + for (; k < k1; k += svl) { const int dk = static_cast<int>(sme_min(k1 - k, Index(svl))); const svbool_t pg_d = Traits::whilelt(k, k1); int r0 = 0; @@ -466,12 +431,12 @@ Vec v1 = sme_read_ver_za<1>(zero, pg_all, uint32_t(c)); EIGEN_IF_CONSTEXPR (NegateOddRows) { if (((k + Index(c)) & Index(1)) != Index(0)) { - v0 = sme_neg(pg_all, v0); - v1 = sme_neg(pg_all, v1); + v0 = pnegate(pg_all, v0); + v1 = pnegate(pg_all, v1); } } - sme_st1(pg_all, &dst[(k + c) * width + r0], v0); - sme_st1(pg_all, &dst[(k + c) * width + r0 + svl], v1); + pstoreu(pg_all, &dst[(k + c) * width + r0], v0); + pstoreu(pg_all, &dst[(k + c) * width + r0 + svl], v1); } } // Trailing row-groups (at most two svl-wide passes remain, since the pair @@ -487,9 +452,9 @@ for (int c = 0; c < dk; ++c) { Vec v0 = sme_read_ver_za<0>(zero, pg_r, uint32_t(c)); EIGEN_IF_CONSTEXPR (NegateOddRows) { - if (((k + Index(c)) & Index(1)) != Index(0)) v0 = sme_neg(pg_r, v0); + if (((k + Index(c)) & Index(1)) != Index(0)) v0 = pnegate(pg_r, v0); } - sme_st1(pg_r, &dst[(k + c) * width + r0], v0); + pstoreu(pg_r, &dst[(k + c) * width + r0], v0); } } } @@ -589,6 +554,25 @@ tail); } +// Tail panel of a deep block through the ZA transposer (sme_transpose_pack_partial); shallow ones and those of at +// most 4 columns keep the NEON tail_transpose_pack, which is faster there. +template <bool Conjugate, typename Scalar, typename Index> +__arm_locally_streaming __arm_new("za") static void sme_tail_pack_streaming(Scalar* dst_panel, const Scalar* src, + Index src_stride, Index depth, int tail) { + sme_transpose_pack<Conjugate>(dst_panel, src, src_stride, depth, tail); +} +template <bool Conjugate, typename Scalar, typename Index> +static EIGEN_ALWAYS_INLINE void tail_pack(Scalar* dst_panel, const Scalar* src, Index src_stride, Index depth, + Index tail) { + EIGEN_IF_CONSTEXPR (!NumTraits<Scalar>::IsComplex) { + if (tail > 4 && depth >= Index(4 * sme_block<Scalar>::nr)) { + sme_tail_pack_streaming<Conjugate>(dst_panel, src, src_stride, depth, static_cast<int>(tail)); + return; + } + } + tail_transpose_pack<Conjugate>(dst_panel, src, src_stride, depth, tail); +} + // De-interleaving load and interleaving store of PS complex values. static EIGEN_ALWAYS_INLINE void sme_neon_ld2(const float* p, Packet4f& re, Packet4f& im) { const float32x4x2_t v = vld2q_f32(p); @@ -947,6 +931,7 @@ if (sme_pack_with_neon<Scalar>(PanelMode ? stride : depth, n)) { neon(block, src, m.stride(), depth, n, stride, offset); } else { + sme_fpsr_guard fpsr; direct(block, src, m.stride(), depth, n, stride, offset); } } else { @@ -1092,12 +1077,18 @@ } } - __arm_locally_streaming static void pack_direct(Scalar* dst_base, const Scalar* EIGEN_RESTRICT src, Index src_stride, - Index depth, Index rows, Index dst_stride, Index dst_offset) { + // EIGEN_DONT_INLINE: GCC (14.1 through trunk) may inline a __arm_locally_streaming function into a + // non-streaming caller and drop its mode switch. The other streaming entry points are __arm_new("za"), + // which GCC does not inline. + __arm_locally_streaming EIGEN_DONT_INLINE static void pack_direct(Scalar* dst_base, const Scalar* EIGEN_RESTRICT src, + Index src_stride, Index depth, Index rows, + Index dst_stride, Index dst_offset) { const Index peeled_rows = (rows / MR) * MR; + const Index i0 = sme_pack_quad_panels<PanelMode>(dst_base, src, src_stride, depth, peeled_rows, Index(MR), + dst_stride, dst_offset); // Full panels of width MR, streamed in svl-wide predicated chunks. - for (Index i = 0; i < peeled_rows; i += MR) { + for (Index i = i0; i < peeled_rows; i += MR) { Scalar* dst_panel = PanelMode ? dst_base + i * dst_stride + dst_offset * MR : dst_base + i * depth; sve_copy_panel<Conjugate>(dst_panel, src + i, src_stride, depth, MR); } @@ -1163,16 +1154,12 @@ pack_full_panels(dst_base, src, src_stride, depth, peeled_rows, dst_stride, dst_offset); } - // Row tail (rows - peeled_rows in [1, MR-1]). This branch runs at most - // once per pack_lhs call with < MR rows and would need a partial-ZA-tile - // dance to vectorise; total copies are < MR * depth per call, which is - // noise vs the main packer's workload, so scalar is the simple choice -- - // taken outside the streaming region above (see tail_transpose_pack). + // Row tail (rows - peeled_rows in [1, MR-1]), at most once per call: see tail_pack. if (peeled_rows < rows) { const Index tail = rows - peeled_rows; Scalar* dst_panel = PanelMode ? dst_base + peeled_rows * dst_stride + dst_offset * tail : dst_base + peeled_rows * depth; - tail_transpose_pack<Conjugate>(dst_panel, src + peeled_rows * src_stride, src_stride, depth, tail); + tail_pack<Conjugate>(dst_panel, src + peeled_rows * src_stride, src_stride, depth, tail); } } @@ -1230,15 +1217,12 @@ pack_full_panels(dst_base, src, src_stride, depth, peeled_cols, dst_stride, dst_offset); } - // Col tail (cols - peeled_cols in [1, NR-1]). Same reasoning as the LHS - // RowMajor packer's row tail: runs at most once per call, < NR cols, not - // worth the partial-ZA-tile handling, and taken outside the streaming - // region above (see tail_transpose_pack). + // Col tail (cols - peeled_cols in [1, NR-1]), at most once per call: see tail_pack. if (peeled_cols < cols) { const Index tail = cols - peeled_cols; Scalar* dst_panel = PanelMode ? dst_base + peeled_cols * dst_stride + dst_offset * tail : dst_base + peeled_cols * depth; - tail_transpose_pack<Conjugate>(dst_panel, src + peeled_cols * src_stride, src_stride, depth, tail); + tail_pack<Conjugate>(dst_panel, src + peeled_cols * src_stride, src_stride, depth, tail); } } @@ -1270,8 +1254,10 @@ } } - __arm_locally_streaming static void pack_direct(Scalar* dst_base, const Scalar* EIGEN_RESTRICT src, Index src_stride, - Index depth, Index cols, Index dst_stride, Index dst_offset) { + // EIGEN_DONT_INLINE: as in sme_pack_lhs_colmajor::pack_direct. + __arm_locally_streaming EIGEN_DONT_INLINE static void pack_direct(Scalar* dst_base, const Scalar* EIGEN_RESTRICT src, + Index src_stride, Index depth, Index cols, + Index dst_stride, Index dst_offset) { const Index peeled_cols = (cols / NR) * NR; for (Index j = 0; j < peeled_cols; j += NR) { @@ -1341,8 +1327,8 @@ EIGEN_ALWAYS_INLINE void sme_store_za_tile(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col, Scalar alpha, Index row_start, int pw, Index col_start, int cw) __arm_streaming __arm_inout("za") { - using Traits = sme_traits<Scalar>; - using Vec = typename Traits::Vec; + using Traits = sme_packet_traits<Scalar>; + using Vec = typename Traits::type; const svbool_t pg_m = Traits::whilelt(0, pw); const svbool_t pg_n = Traits::whilelt(0, cw); // FMLA and FADD have equal latency/throughput on ARMv9 cores, and @@ -1351,8 +1337,8 @@ // keeps the store compact and measures no worse (and a few percent // better on small matrices, where the branch would otherwise disrupt // instruction scheduling). - const Vec vzero = Traits::dup(Scalar(0)); - const Vec valpha = Traits::dup(alpha); + const Vec vzero = pset1<Vec>(Scalar(0)); + const Vec valpha = pset1<Vec>(alpha); // Two C slices are loaded before either is stored: a C line the caller wrote // from non-streaming code just before the kernel does not forward across the @@ -1365,15 +1351,15 @@ for (; ci + 2 <= cw; ci += 2) { Scalar* p0 = C + row_start + (col_start + ci) * C_stride_col; Scalar* p1 = p0 + C_stride_col; - Vec c0 = sme_ld1(pg_m, p0); - Vec c1 = sme_ld1(pg_m, p1); - sme_st1(pg_m, p0, sme_mla(pg_m, c0, sme_read_ver_za<TileId>(vzero, pg_m, (uint32_t)ci), valpha)); - sme_st1(pg_m, p1, sme_mla(pg_m, c1, sme_read_ver_za<TileId>(vzero, pg_m, (uint32_t)(ci + 1)), valpha)); + Vec c0 = ploadu(pg_m, p0); + Vec c1 = ploadu(pg_m, p1); + pstoreu(pg_m, p0, pmadd(pg_m, sme_read_ver_za<TileId>(vzero, pg_m, (uint32_t)ci), valpha, c0)); + pstoreu(pg_m, p1, pmadd(pg_m, sme_read_ver_za<TileId>(vzero, pg_m, (uint32_t)(ci + 1)), valpha, c1)); } if (ci < cw) { Scalar* pC = C + row_start + (col_start + ci) * C_stride_col; - Vec vc = sme_ld1(pg_m, pC); - sme_st1(pg_m, pC, sme_mla(pg_m, vc, sme_read_ver_za<TileId>(vzero, pg_m, (uint32_t)ci), valpha)); + Vec vc = ploadu(pg_m, pC); + pstoreu(pg_m, pC, pmadd(pg_m, sme_read_ver_za<TileId>(vzero, pg_m, (uint32_t)ci), valpha, vc)); } } else if (C_stride_col == 1) { // Row-major C: extract horizontal slices (rows of the ZA tile) @@ -1381,15 +1367,15 @@ for (; ri + 2 <= pw; ri += 2) { Scalar* p0 = C + (row_start + ri) * C_stride_row + col_start; Scalar* p1 = p0 + C_stride_row; - Vec c0 = sme_ld1(pg_n, p0); - Vec c1 = sme_ld1(pg_n, p1); - sme_st1(pg_n, p0, sme_mla(pg_n, c0, sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)ri), valpha)); - sme_st1(pg_n, p1, sme_mla(pg_n, c1, sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)(ri + 1)), valpha)); + Vec c0 = ploadu(pg_n, p0); + Vec c1 = ploadu(pg_n, p1); + pstoreu(pg_n, p0, pmadd(pg_n, sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)ri), valpha, c0)); + pstoreu(pg_n, p1, pmadd(pg_n, sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)(ri + 1)), valpha, c1)); } if (ri < pw) { Scalar* pC = C + (row_start + ri) * C_stride_row + col_start; - Vec vc = sme_ld1(pg_n, pC); - sme_st1(pg_n, pC, sme_mla(pg_n, vc, sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)ri), valpha)); + Vec vc = ploadu(pg_n, pC); + pstoreu(pg_n, pC, pmadd(pg_n, sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)ri), valpha, vc)); } } else { // General stride: extract rows to temp buffer, scatter to C. scratch @@ -1399,8 +1385,8 @@ Scalar scratch[sme_block<Scalar>::nr]; for (int ri = 0; ri < pw; ++ri) { Vec vres = sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)ri); - vres = sme_mul(pg_n, vres, valpha); - sme_st1(pg_n, scratch, vres); + vres = pmul(pg_n, vres, valpha); + pstoreu(pg_n, scratch, vres); for (int ci = 0; ci < cw; ++ci) { C[(row_start + ri) * C_stride_row + (col_start + ci) * C_stride_col] += scratch[ci]; } @@ -1420,11 +1406,43 @@ * pattern repeats across blocks). *****************************************************************************/ +// Full 2*svl x 2*svl block into column-major C: four columns per vertical four-slice read of the two tiles that +// share them, each column loaded and stored whole with two-vector accesses, all four loaded before any is stored. +template <int TileLo, int TileHi, typename Scalar, typename Index> +EIGEN_ALWAYS_INLINE void sme_store_tile_pair_colmajor(Scalar* EIGEN_RESTRICT C, Index ldc, Scalar alpha, + int svl) __arm_streaming __arm_inout("za") { + using Traits = sme_packet_traits<Scalar>; + const svcount_t pn = Traits::ptrue_c(); + const svbool_t pg = Traits::ptrue(); + const typename Traits::type valpha = pset1<typename Traits::type>(alpha); + for (int c = 0; c < svl; c += 4) { + const auto lo = sme_read_ver_za_vg4<TileLo, Scalar>(uint32_t(c)); + const auto hi = sme_read_ver_za_vg4<TileHi, Scalar>(uint32_t(c)); + Scalar* p = C + Index(c) * ldc; + const auto c0 = ploadu_x2(pn, p), c1 = ploadu_x2(pn, p + ldc), c2 = ploadu_x2(pn, p + 2 * ldc), + c3 = ploadu_x2(pn, p + 3 * ldc); + pstoreu_x2(pn, p, + pcreate(pmadd(pg, pget<0>(lo), valpha, pget<0>(c0)), pmadd(pg, pget<0>(hi), valpha, pget<1>(c0)))); + pstoreu_x2(pn, p + ldc, + pcreate(pmadd(pg, pget<1>(lo), valpha, pget<0>(c1)), pmadd(pg, pget<1>(hi), valpha, pget<1>(c1)))); + pstoreu_x2(pn, p + 2 * ldc, + pcreate(pmadd(pg, pget<2>(lo), valpha, pget<0>(c2)), pmadd(pg, pget<2>(hi), valpha, pget<1>(c2)))); + pstoreu_x2(pn, p + 3 * ldc, + pcreate(pmadd(pg, pget<3>(lo), valpha, pget<0>(c3)), pmadd(pg, pget<3>(hi), valpha, pget<1>(c3)))); + } +} + template <typename Scalar, typename Index> EIGEN_ALWAYS_INLINE void sme_store_2x2_grid(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col, Scalar alpha, Index row_start, int rlo, int rhi, Index col_start, int clo, int chi) __arm_streaming __arm_inout("za") { - const int svl = sme_traits<Scalar>::svl(); + const int svl = sme_packet_traits<Scalar>::size(); + if (svl >= 4 && C_stride_row == 1 && rlo == svl && rhi == svl && clo == svl && chi == svl) { + Scalar* c0 = C + row_start + col_start * C_stride_col; + sme_store_tile_pair_colmajor<0, 2>(c0, C_stride_col, alpha, svl); + sme_store_tile_pair_colmajor<1, 3>(c0 + Index(svl) * C_stride_col, C_stride_col, alpha, svl); + return; + } sme_store_za_tile<Scalar, 0>(C, C_stride_row, C_stride_col, alpha, row_start, rlo, col_start, clo); if (chi > 0) { sme_store_za_tile<Scalar, 1>(C, C_stride_row, C_stride_col, alpha, row_start, rlo, col_start + svl, chi); @@ -1444,9 +1462,10 @@ // factoring it out is identical to the inline form. template <typename Scalar> static EIGEN_ALWAYS_INLINE void outer_product_2x2( - typename sme_traits<Scalar>::Vec a_lo, typename sme_traits<Scalar>::Vec a_hi, typename sme_traits<Scalar>::Vec b_lo, - typename sme_traits<Scalar>::Vec b_hi) __arm_streaming __arm_inout("za") { - const svbool_t all = sme_traits<Scalar>::ptrue(); + typename sme_packet_traits<Scalar>::type a_lo, typename sme_packet_traits<Scalar>::type a_hi, + typename sme_packet_traits<Scalar>::type b_lo, + typename sme_packet_traits<Scalar>::type b_hi) __arm_streaming __arm_inout("za") { + const svbool_t all = sme_packet_traits<Scalar>::ptrue(); sme_mopa<0>(all, all, a_lo, b_lo); sme_mopa<1>(all, all, a_lo, b_hi); sme_mopa<2>(all, all, a_hi, b_lo); @@ -1466,9 +1485,11 @@ * (FMOPA) or subtract (FMOPS) -- a compile-time choice, with no work in the * depth loop and no separate conjugating packer. * - * Slices come back out through sme_read_scaled_slice, which applies the complex + * Slices come back out through sme_read_slice, which applies the complex * alpha and interleaves the halves with ZIP1/ZIP2 into the two vectors that - * cover one slice's worth of contiguous complex results. + * cover one slice's worth of contiguous complex results. The kernel that + * folds a real alpha keeps them deinterleaved instead (see + * sme_accumulate_pair_real_alpha). * * A ZA tile number is an instruction immediate, so cells outside the tile grid * (complex<float> has two tile pairs, hence a single grid row) are dropped by @@ -1482,12 +1503,12 @@ // (see sme_store_za_pair) -- streaming-mode FP is de-rated enough on Apple M4 // that those four cost about as much as the rest of the slice. template <typename RealScalar, int TileRe, int TileIm, bool Vertical, bool ScaleByAlpha> -EIGEN_ALWAYS_INLINE void sme_read_slice(svbool_t pg, typename sme_traits<RealScalar>::Vec valpha_re, - typename sme_traits<RealScalar>::Vec valpha_im, - typename sme_traits<RealScalar>::Vec vzero, uint32_t slice, - typename sme_traits<RealScalar>::Vec& lo, - typename sme_traits<RealScalar>::Vec& hi) __arm_streaming __arm_inout("za") { - using Vec = typename sme_traits<RealScalar>::Vec; +EIGEN_ALWAYS_INLINE void sme_read_slice( + svbool_t pg, typename sme_packet_traits<RealScalar>::type valpha_re, + typename sme_packet_traits<RealScalar>::type valpha_im, typename sme_packet_traits<RealScalar>::type vzero, + uint32_t slice, typename sme_packet_traits<RealScalar>::type& lo, + typename sme_packet_traits<RealScalar>::type& hi) __arm_streaming __arm_inout("za") { + using Vec = typename sme_packet_traits<RealScalar>::type; Vec re, im; EIGEN_IF_CONSTEXPR (Vertical) { re = sme_read_ver_za<TileRe>(vzero, pg, slice); @@ -1497,13 +1518,13 @@ im = sme_read_hor_za<TileIm>(vzero, pg, slice); } EIGEN_IF_CONSTEXPR (ScaleByAlpha) { - const Vec out_re = sme_mls(pg, sme_mul(pg, re, valpha_re), im, valpha_im); - const Vec out_im = sme_mla(pg, sme_mul(pg, im, valpha_re), re, valpha_im); - lo = sme_zip1(out_re, out_im); - hi = sme_zip2(out_re, out_im); + const Vec out_re = pnmadd(pg, im, valpha_im, pmul(pg, re, valpha_re)); + const Vec out_im = pmadd(pg, re, valpha_im, pmul(pg, im, valpha_re)); + lo = pzip1(out_re, out_im); + hi = pzip2(out_re, out_im); } else { - lo = sme_zip1(re, im); - hi = sme_zip2(re, im); + lo = pzip1(re, im); + hi = pzip2(re, im); } } @@ -1514,32 +1535,87 @@ template <typename RealScalar, int TileRe, int TileIm, bool Vertical, bool ScaleByAlpha, typename Index> EIGEN_ALWAYS_INLINE void sme_accumulate_pair_impl( RealScalar* EIGEN_RESTRICT p, Index step, int slices, int lanes, svbool_t pg, - typename sme_traits<RealScalar>::Vec valpha_re, typename sme_traits<RealScalar>::Vec valpha_im, - typename sme_traits<RealScalar>::Vec vzero) __arm_streaming __arm_inout("za") { - using Traits = sme_traits<RealScalar>; - using Vec = typename Traits::Vec; - const int svl = Traits::svl(); + typename sme_packet_traits<RealScalar>::type valpha_re, typename sme_packet_traits<RealScalar>::type valpha_im, + typename sme_packet_traits<RealScalar>::type vzero) __arm_streaming __arm_inout("za") { + using Traits = sme_packet_traits<RealScalar>; + using Vec = typename Traits::type; + const int svl = Traits::size(); const svbool_t pl0 = Traits::whilelt(0, lanes); const svbool_t pl1 = Traits::whilelt(svl, lanes); for (int s = 0; s < slices; ++s, p += step) { Vec lo, hi; sme_read_slice<RealScalar, TileRe, TileIm, Vertical, ScaleByAlpha>(pg, valpha_re, valpha_im, vzero, uint32_t(s), lo, hi); - sme_st1(pl0, p, sme_add(pl0, sme_ld1(pl0, p), lo)); + pstoreu(pl0, p, padd(pl0, ploadu(pl0, p), lo)); // pl1 is all-false when one vector covers the slice, and an inactive lane // neither reads nor writes -- so this needs no `lanes > svl` guard, only an // address the destination is allowed to form. RealScalar* EIGEN_RESTRICT phi = sme_offset(p, Index(svl)); - sme_st1(pl1, phi, sme_add(pl1, sme_ld1(pl1, phi), hi)); + pstoreu(pl1, phi, padd(pl1, ploadu(pl1, phi), hi)); } } +// A real alpha on slices spanning two vectors, in the FoldRealAlpha kernel: +// LD2/ST2 keep C deinterleaved with one predicate lane per complex result, so +// each half is one FMA, c + alpha*acc, as in the real-scalar store. A single +// FMA rounds once, so it overflows only when the exact result does. This +// covers C.noalias() -= A*B, whose alpha is -1. +// +// `limit`, the slices left in the C block from the first one, bounds the +// prefetch to the block (unbounded, it measured 0.45-0.5x on 32^3 and 64^3). +// The prefetch runs 8 slices ahead of the C load that heads each FMA; without +// it the fold ran at 0.94x of that at 1024^3. The barrier keeps the prefetch +// address setup inside its branch. +// +// A complex alpha keeps sme_accumulate_pair_impl, which forms alpha*acc before +// adding C. Folding C into either of its two FMAs, e.g. (c_re + re*ar) - im*ai, +// saves two ops per slice but overflows when C and the first product exceed +// the range before the second cancels them (!3139 review: acc = (h,h), +// alpha = (1,1), C = (3h,0)). On Apple M4, LD2/ST2 measured no gain with +// unchanged arithmetic and 0.68-0.75x on one-vector slices, and FCMLA 0.70x +// (#3129). template <typename RealScalar, int TileRe, int TileIm, bool Vertical, typename Index> +EIGEN_ALWAYS_INLINE void sme_accumulate_pair_real_alpha( + RealScalar* EIGEN_RESTRICT p, Index step, int slices, Index limit, svbool_t pg, + typename sme_packet_traits<RealScalar>::type valpha, + typename sme_packet_traits<RealScalar>::type vzero) __arm_streaming __arm_inout("za") { + using Traits = sme_packet_traits<RealScalar>; + using Vec = typename Traits::type; + const int svl = Traits::size(); + constexpr int kPrefetchSlices = 8; + for (int s = 0; s < slices; ++s, p += step) { + if (s + kPrefetchSlices < limit) { + RealScalar* pf = p; + EIGEN_OPTIMIZATION_BARRIER(pf) + pf = sme_offset(pf, Index(kPrefetchSlices) * step); + __builtin_prefetch(pf, 1, 3); + __builtin_prefetch(sme_offset(pf, Index(svl)), 1, 3); + } + Vec re, im; + EIGEN_IF_CONSTEXPR (Vertical) { + re = sme_read_ver_za<TileRe>(vzero, pg, uint32_t(s)); + im = sme_read_ver_za<TileIm>(vzero, pg, uint32_t(s)); + } else { + re = sme_read_hor_za<TileRe>(vzero, pg, uint32_t(s)); + im = sme_read_hor_za<TileIm>(vzero, pg, uint32_t(s)); + } + const typename Traits::type_x2 c = pld2(pg, p); + pst2(pg, p, pcreate(pmadd(pg, re, valpha, pget<0>(c)), pmadd(pg, im, valpha, pget<1>(c)))); + } +} + +// FoldRealAlpha selects the kernel instantiation. The folding kernel only +// sees a real alpha other than 1 (see sme_gebp_dispatch); the other kernel +// keeps exactly the unscaled and complex-alpha paths, so the fold's code never +// reaches products that cannot use it. +template <typename RealScalar, int TileRe, int TileIm, bool Vertical, bool FoldRealAlpha, typename Index> EIGEN_ALWAYS_INLINE void sme_accumulate_pair( - bool scale_by_alpha, RealScalar* EIGEN_RESTRICT p, Index step, int slices, int lanes, svbool_t pg, - typename sme_traits<RealScalar>::Vec valpha_re, typename sme_traits<RealScalar>::Vec valpha_im, - typename sme_traits<RealScalar>::Vec vzero) __arm_streaming __arm_inout("za") { - if (scale_by_alpha) { + bool scale_by_alpha, RealScalar* EIGEN_RESTRICT p, Index step, int slices, Index limit, int lanes, svbool_t pg, + typename sme_packet_traits<RealScalar>::type valpha_re, typename sme_packet_traits<RealScalar>::type valpha_im, + typename sme_packet_traits<RealScalar>::type vzero) __arm_streaming __arm_inout("za") { + if (FoldRealAlpha && lanes > sme_packet_traits<RealScalar>::size()) { + sme_accumulate_pair_real_alpha<RealScalar, TileRe, TileIm, Vertical>(p, step, slices, limit, pg, valpha_re, vzero); + } else if (scale_by_alpha) { sme_accumulate_pair_impl<RealScalar, TileRe, TileIm, Vertical, true>(p, step, slices, lanes, pg, valpha_re, valpha_im, vzero); } else { @@ -1550,26 +1626,29 @@ // Store one complex tile pair back to C. `pw` is the row-predicate width for // this cell and `cw` the column one, both <= the runtime svl. -template <typename RealScalar, int TileRe, int TileIm, typename Index> +template <typename RealScalar, int TileRe, int TileIm, bool FoldRealAlpha, typename Index> EIGEN_ALWAYS_INLINE void sme_store_za_pair(std::complex<RealScalar>* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col, std::complex<RealScalar> alpha, Index row_start, int pw, - Index col_start, int cw) __arm_streaming __arm_inout("za") { + Index col_start, int cw, Index rows, + Index cols) __arm_streaming __arm_inout("za") { using Scalar = std::complex<RealScalar>; - using Traits = sme_traits<RealScalar>; - using Vec = typename Traits::Vec; - const int svl = Traits::svl(); + using Traits = sme_packet_traits<RealScalar>; + using Vec = typename Traits::type; + const int svl = Traits::size(); const svbool_t pg_m = Traits::whilelt(0, pw); const svbool_t pg_n = Traits::whilelt(0, cw); - const Vec vzero = Traits::dup(RealScalar(0)); + const Vec vzero = pset1<Vec>(RealScalar(0)); // std::complex's accessors are ordinary functions, which clang cannot inline // into a streaming context; the resulting mode switch would sit in this loop. // A complex is layout-compatible with its two-element real array, so read the // parts through that view instead. const RealScalar* alpha_parts = reinterpret_cast<const RealScalar*>(&alpha); - const Vec valpha_re = Traits::dup(alpha_parts[0]); - const Vec valpha_im = Traits::dup(alpha_parts[1]); + const Vec valpha_re = pset1<Vec>(alpha_parts[0]); + const Vec valpha_im = pset1<Vec>(alpha_parts[1]); // Scaling by 1 + 0i is exact, so skipping it is bit-identical -- and it is by // far the common case, since a plain product carries alpha = 1. + // With FoldRealAlpha, sme_gebp_dispatch has already established that alpha + // is real and not 1. const bool scale = !(alpha_parts[0] == RealScalar(1) && alpha_parts[1] == RealScalar(0)); RealScalar* EIGEN_RESTRICT rC = reinterpret_cast<RealScalar*>(C); @@ -1577,13 +1656,13 @@ // Column-major C: vertical slices are the tile pair's columns, and one // slice is pw contiguous complex results, i.e. 2*pw reals. RealScalar* p = rC + Index(2) * (row_start + col_start * C_stride_col); - sme_accumulate_pair<RealScalar, TileRe, TileIm, true>(scale, p, Index(2) * C_stride_col, cw, 2 * pw, pg_m, - valpha_re, valpha_im, vzero); + sme_accumulate_pair<RealScalar, TileRe, TileIm, true, FoldRealAlpha>( + scale, p, Index(2) * C_stride_col, cw, cols - col_start, 2 * pw, pg_m, valpha_re, valpha_im, vzero); } else if (C_stride_col == 1) { // Row-major C: horizontal slices are the tile pair's rows. RealScalar* p = rC + Index(2) * (row_start * C_stride_row + col_start); - sme_accumulate_pair<RealScalar, TileRe, TileIm, false>(scale, p, Index(2) * C_stride_row, pw, 2 * cw, pg_n, - valpha_re, valpha_im, vzero); + sme_accumulate_pair<RealScalar, TileRe, TileIm, false, FoldRealAlpha>( + scale, p, Index(2) * C_stride_row, pw, rows - row_start, 2 * cw, pg_n, valpha_re, valpha_im, vzero); } else { // General stride: interleave a row into a temp buffer, scatter to C. Every // caller passes cw <= min(svl, nr), so nr is a static bound on the buffer, @@ -1597,10 +1676,10 @@ for (int ri = 0; ri < pw; ++ri) { Vec lo, hi; sme_read_slice<RealScalar, TileRe, TileIm, false, true>(pg_n, valpha_re, valpha_im, vzero, uint32_t(ri), lo, hi); - sme_st1(pl0, rscratch, lo); + pstoreu(pl0, rscratch, lo); // scratch is nr complex, i.e. 2*nr >= 2*svl reals, so rscratch + svl is // always in bounds; pl1 is all-false when one vector already covers cw. - sme_st1(pl1, rscratch + svl, hi); + pstoreu(pl1, rscratch + svl, hi); for (int ci = 0; ci < cw; ++ci) { C[(row_start + ri) * C_stride_row + (col_start + ci) * C_stride_col] += scratch[ci]; } @@ -1614,7 +1693,7 @@ bool InGrid = (R < sme_block<Scalar>::kGridRows && C < sme_block<Scalar>::kGridCols)> struct sme_complex_cell { using RealScalar = typename NumTraits<Scalar>::Real; - using Vec = typename sme_traits<RealScalar>::Vec; + using Vec = typename sme_packet_traits<RealScalar>::type; static constexpr int kTileRe = 2 * (R * sme_block<Scalar>::kGridCols + C); static constexpr int kTileIm = kTileRe + 1; @@ -1626,23 +1705,23 @@ sme_mopa_signed<kTileIm, ConjLhs>(pm, pn, a_im, b_re); } - template <typename Index> + template <bool FoldRealAlpha, typename Index> static EIGEN_ALWAYS_INLINE void store(Scalar* EIGEN_RESTRICT dst, Index C_stride_row, Index C_stride_col, - Scalar alpha, Index row_start, int pw, Index col_start, - int cw) __arm_streaming __arm_inout("za") { - sme_store_za_pair<RealScalar, kTileRe, kTileIm>(dst, C_stride_row, C_stride_col, alpha, row_start, pw, col_start, - cw); + Scalar alpha, Index row_start, int pw, Index col_start, int cw, Index rows, + Index cols) __arm_streaming __arm_inout("za") { + sme_store_za_pair<RealScalar, kTileRe, kTileIm, FoldRealAlpha>(dst, C_stride_row, C_stride_col, alpha, row_start, + pw, col_start, cw, rows, cols); } }; template <typename Scalar, int R, int C, bool ConjLhs, bool ConjRhs> struct sme_complex_cell<Scalar, R, C, ConjLhs, ConjRhs, false> { - using Vec = typename sme_traits<typename NumTraits<Scalar>::Real>::Vec; + using Vec = typename sme_packet_traits<typename NumTraits<Scalar>::Real>::type; static EIGEN_ALWAYS_INLINE void accumulate(svbool_t, svbool_t, Vec, Vec, Vec, Vec) __arm_streaming __arm_inout("za") { } - template <typename Index> - static EIGEN_ALWAYS_INLINE void store(Scalar*, Index, Index, Scalar, Index, int, Index, - int) __arm_streaming __arm_inout("za") {} + template <bool FoldRealAlpha, typename Index> + static EIGEN_ALWAYS_INLINE void store(Scalar*, Index, Index, Scalar, Index, int, Index, int, Index, + Index) __arm_streaming __arm_inout("za") {} }; /***************************************************************************** @@ -1665,12 +1744,14 @@ template <bool ConjLhs, bool ConjRhs, typename Scalar, typename Index> EIGEN_ALWAYS_INLINE void sme_process(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col, const Scalar* EIGEN_RESTRICT blA, const Scalar* EIGEN_RESTRICT blB, Index depth, - Scalar alpha, Index row_start, int pw, Index col_start, - int cw) __arm_streaming __arm_inout("za") { + Scalar alpha, Index row_start, int pw, Index col_start, int cw, + Index a_step) __arm_streaming __arm_inout("za") { // Conjugation is the identity on real scalars, so this overload ignores it. - using Traits = sme_traits<Scalar>; - using Vec = typename Traits::Vec; - const int svl = Traits::svl(); + // a_step is the distance between A's depth steps: pw for a packed panel, the + // column stride for a ColMajor source read in place. + using Traits = sme_packet_traits<Scalar>; + using Vec = typename Traits::type; + const int svl = Traits::size(); for (int rt = 0; rt < pw; rt += 2 * svl) { const int rpw = sme_min(pw - rt, 2 * svl); @@ -1695,36 +1776,52 @@ const svcount_t pn = Traits::ptrue_c(); const Index depth_4 = (depth / 4) * 4; Index k = 0; - for (; k < depth_4; k += 4) { - typename Traits::Vec4 va_01 = sme_ld1_x4(pn, &blA[k * pw]); - typename Traits::Vec4 vb_01 = sme_ld1_x4(pn, &blB[k * cw]); + if (a_step == Index(pw)) { + for (; k < depth_4; k += 4) { + typename Traits::type_x4 va_01 = ploadu_x4(pn, &blA[k * pw]); + typename Traits::type_x4 vb_01 = ploadu_x4(pn, &blB[k * cw]); - // d0 - outer_product_2x2<Scalar>(sme_get<0>(va_01), sme_get<1>(va_01), sme_get<0>(vb_01), sme_get<1>(vb_01)); - // d1 - outer_product_2x2<Scalar>(sme_get<2>(va_01), sme_get<3>(va_01), sme_get<2>(vb_01), sme_get<3>(vb_01)); + // d0 + outer_product_2x2<Scalar>(pget<0>(va_01), pget<1>(va_01), pget<0>(vb_01), pget<1>(vb_01)); + // d1 + outer_product_2x2<Scalar>(pget<2>(va_01), pget<3>(va_01), pget<2>(vb_01), pget<3>(vb_01)); - typename Traits::Vec4 va_23 = sme_ld1_x4(pn, &blA[(k + 2) * pw]); - typename Traits::Vec4 vb_23 = sme_ld1_x4(pn, &blB[(k + 2) * cw]); + typename Traits::type_x4 va_23 = ploadu_x4(pn, &blA[(k + 2) * pw]); + typename Traits::type_x4 vb_23 = ploadu_x4(pn, &blB[(k + 2) * cw]); - // d2 - outer_product_2x2<Scalar>(sme_get<0>(va_23), sme_get<1>(va_23), sme_get<0>(vb_23), sme_get<1>(vb_23)); - // d3 - outer_product_2x2<Scalar>(sme_get<2>(va_23), sme_get<3>(va_23), sme_get<2>(vb_23), sme_get<3>(vb_23)); + // d2 + outer_product_2x2<Scalar>(pget<0>(va_23), pget<1>(va_23), pget<0>(vb_23), pget<1>(vb_23)); + // d3 + outer_product_2x2<Scalar>(pget<2>(va_23), pget<3>(va_23), pget<2>(vb_23), pget<3>(vb_23)); + } + } else { + // A read in place: its depth steps are a_step apart, one x2 load each. + for (; k < depth_4; k += 4) { + typename Traits::type_x2 va_0 = ploadu_x2(pn, &blA[k * a_step]); + typename Traits::type_x2 va_1 = ploadu_x2(pn, &blA[(k + 1) * a_step]); + typename Traits::type_x4 vb_01 = ploadu_x4(pn, &blB[k * cw]); + outer_product_2x2<Scalar>(pget<0>(va_0), pget<1>(va_0), pget<0>(vb_01), pget<1>(vb_01)); + outer_product_2x2<Scalar>(pget<0>(va_1), pget<1>(va_1), pget<2>(vb_01), pget<3>(vb_01)); + typename Traits::type_x2 va_2 = ploadu_x2(pn, &blA[(k + 2) * a_step]); + typename Traits::type_x2 va_3 = ploadu_x2(pn, &blA[(k + 3) * a_step]); + typename Traits::type_x4 vb_23 = ploadu_x4(pn, &blB[(k + 2) * cw]); + outer_product_2x2<Scalar>(pget<0>(va_2), pget<1>(va_2), pget<0>(vb_23), pget<1>(vb_23)); + outer_product_2x2<Scalar>(pget<0>(va_3), pget<1>(va_3), pget<2>(vb_23), pget<3>(vb_23)); + } } // Depth tail: one x2 load per side per step. for (; k < depth; ++k) { - typename Traits::Vec2 va = sme_ld1_x2(pn, &blA[k * pw]); - typename Traits::Vec2 vb = sme_ld1_x2(pn, &blB[k * cw]); - outer_product_2x2<Scalar>(sme_get<0>(va), sme_get<1>(va), sme_get<0>(vb), sme_get<1>(vb)); + typename Traits::type_x2 va = ploadu_x2(pn, &blA[k * a_step]); + typename Traits::type_x2 vb = ploadu_x2(pn, &blB[k * cw]); + outer_product_2x2<Scalar>(pget<0>(va), pget<1>(va), pget<0>(vb), pget<1>(vb)); } } else { for (Index k = 0; k < depth; ++k) { - Vec a_lo = sme_ld1(pg_rlo, &blA[k * pw + rt]); - Vec b_lo = sme_ld1(pg_clo, &blB[k * cw + ct]); + Vec a_lo = ploadu(pg_rlo, &blA[k * a_step + rt]); + Vec b_lo = ploadu(pg_clo, &blB[k * cw + ct]); - Vec a_hi = sme_ld1(pg_rhi, sme_offset(blA, k * pw + rt + svl)); - Vec b_hi = sme_ld1(pg_chi, sme_offset(blB, k * cw + ct + svl)); + Vec a_hi = ploadu(pg_rhi, sme_offset(blA, k * a_step + rt + svl)); + Vec b_hi = ploadu(pg_chi, sme_offset(blB, k * cw + ct + svl)); sme_mopa<0>(pg_rlo, pg_clo, a_lo, b_lo); if (svptest_any(pg_chi, pg_chi)) sme_mopa<1>(pg_rlo, pg_chi, a_lo, b_hi); @@ -1747,23 +1844,24 @@ * dropped at compile time by sme_complex_cell, so a narrower grid simply never * reaches them (its hi widths are structurally zero). *****************************************************************************/ -template <typename Scalar, bool ConjLhs, bool ConjRhs, typename Index> +template <typename Scalar, bool ConjLhs, bool ConjRhs, bool FoldRealAlpha, typename Index> EIGEN_ALWAYS_INLINE void sme_store_complex_grid(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col, Scalar alpha, Index row_start, int rlo, int rhi, Index col_start, - int clo, int chi) __arm_streaming __arm_inout("za") { - const int svl = sme_traits<typename NumTraits<Scalar>::Real>::svl(); - sme_complex_cell<Scalar, 0, 0, ConjLhs, ConjRhs>::store(C, C_stride_row, C_stride_col, alpha, row_start, rlo, - col_start, clo); + int clo, int chi, Index rows, + Index cols) __arm_streaming __arm_inout("za") { + const int svl = sme_packet_traits<typename NumTraits<Scalar>::Real>::size(); + sme_complex_cell<Scalar, 0, 0, ConjLhs, ConjRhs>::template store<FoldRealAlpha>( + C, C_stride_row, C_stride_col, alpha, row_start, rlo, col_start, clo, rows, cols); if (chi > 0) { - sme_complex_cell<Scalar, 0, 1, ConjLhs, ConjRhs>::store(C, C_stride_row, C_stride_col, alpha, row_start, rlo, - col_start + svl, chi); + sme_complex_cell<Scalar, 0, 1, ConjLhs, ConjRhs>::template store<FoldRealAlpha>( + C, C_stride_row, C_stride_col, alpha, row_start, rlo, col_start + svl, chi, rows, cols); } if (rhi > 0) { - sme_complex_cell<Scalar, 1, 0, ConjLhs, ConjRhs>::store(C, C_stride_row, C_stride_col, alpha, row_start + svl, rhi, - col_start, clo); + sme_complex_cell<Scalar, 1, 0, ConjLhs, ConjRhs>::template store<FoldRealAlpha>( + C, C_stride_row, C_stride_col, alpha, row_start + svl, rhi, col_start, clo, rows, cols); if (chi > 0) { - sme_complex_cell<Scalar, 1, 1, ConjLhs, ConjRhs>::store(C, C_stride_row, C_stride_col, alpha, row_start + svl, - rhi, col_start + svl, chi); + sme_complex_cell<Scalar, 1, 1, ConjLhs, ConjRhs>::template store<FoldRealAlpha>( + C, C_stride_row, C_stride_col, alpha, row_start + svl, rhi, col_start + svl, chi, rows, cols); } } } @@ -1777,16 +1875,18 @@ * are four contiguous predicated loads at a fixed offset apart, and the four * outer products they feed reuse all of them. *****************************************************************************/ -template <bool ConjLhs, bool ConjRhs, typename RealScalar, typename Index> +template <bool ConjLhs, bool ConjRhs, bool FoldRealAlpha, typename RealScalar, typename Index> EIGEN_ALWAYS_INLINE void sme_process(std::complex<RealScalar>* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col, const std::complex<RealScalar>* EIGEN_RESTRICT blA, const std::complex<RealScalar>* EIGEN_RESTRICT blB, Index depth, - std::complex<RealScalar> alpha, Index row_start, int pw, Index col_start, - int cw) __arm_streaming __arm_inout("za") { + std::complex<RealScalar> alpha, Index row_start, int pw, Index col_start, int cw, + Index lhs_step, Index rows, Index cols) __arm_streaming __arm_inout("za") { + // Complex panels are always packed (split real/imaginary halves): lhs_step is pw. + EIGEN_UNUSED_VARIABLE(lhs_step); using Scalar = std::complex<RealScalar>; - using Traits = sme_traits<RealScalar>; - using Vec = typename Traits::Vec; - const int svl = Traits::svl(); + using Traits = sme_packet_traits<RealScalar>; + using Vec = typename Traits::type; + const int svl = Traits::size(); constexpr int GridRows = sme_block<Scalar>::kGridRows; constexpr int GridCols = sme_block<Scalar>::kGridCols; using Cell00 = sme_complex_cell<Scalar, 0, 0, ConjLhs, ConjRhs>; @@ -1822,14 +1922,14 @@ // Grouping them lets the compiler issue them in parallel, and gating the // outer products on svptest_any keeps the depth loop unswitched without // materialising the r1/c1 counts here. - const Vec a0_re = sme_ld1(pg_r0, pa); - const Vec a0_im = sme_ld1(pg_r0, pa + pw); - const Vec b0_re = sme_ld1(pg_c0, pb); - const Vec b0_im = sme_ld1(pg_c0, pb + cw); - const Vec a1_re = sme_ld1(pg_r1, sme_offset(pa, Index(svl))); - const Vec a1_im = sme_ld1(pg_r1, sme_offset(pa, Index(pw) + Index(svl))); - const Vec b1_re = sme_ld1(pg_c1, sme_offset(pb, Index(svl))); - const Vec b1_im = sme_ld1(pg_c1, sme_offset(pb, Index(cw) + Index(svl))); + const Vec a0_re = ploadu(pg_r0, pa); + const Vec a0_im = ploadu(pg_r0, pa + pw); + const Vec b0_re = ploadu(pg_c0, pb); + const Vec b0_im = ploadu(pg_c0, pb + cw); + const Vec a1_re = ploadu(pg_r1, sme_offset(pa, Index(svl))); + const Vec a1_im = ploadu(pg_r1, sme_offset(pa, Index(pw) + Index(svl))); + const Vec b1_re = ploadu(pg_c1, sme_offset(pb, Index(svl))); + const Vec b1_im = ploadu(pg_c1, sme_offset(pb, Index(cw) + Index(svl))); Cell00::accumulate(pg_r0, pg_c0, a0_re, a0_im, b0_re, b0_im); if (svptest_any(pg_c1, pg_c1)) { Cell01::accumulate(pg_r0, pg_c1, a0_re, a0_im, b1_re, b1_im); @@ -1842,13 +1942,195 @@ } } - sme_store_complex_grid<Scalar, ConjLhs, ConjRhs>(C, C_stride_row, C_stride_col, alpha, row_start + rt, r0, r1, - col_start + ct, c0, c1); + sme_store_complex_grid<Scalar, ConjLhs, ConjRhs, FoldRealAlpha>( + C, C_stride_row, C_stride_col, alpha, row_start + rt, r0, r1, col_start + ct, c0, c1, rows, cols); } } } -template <typename Scalar, bool ConjLhs, bool ConjRhs, typename Index> +// A RHS panel of at most svl columns fills only one tile column of the 2 x 2 grid, so the four tiles are stacked along +// M instead: a full LHS panel (tiles 0 and 1) and the next one (tiles 2 and 3, pw1 rows) against the same B vector. +template <typename Scalar, typename Index> +EIGEN_ALWAYS_INLINE void sme_process_narrow(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col, + const Scalar* EIGEN_RESTRICT blA0, Index step0, + const Scalar* EIGEN_RESTRICT blA1, Index step1, int pw1, + const Scalar* EIGEN_RESTRICT blB, int cw, Index depth, Scalar alpha, + Index row_start, Index col_start) __arm_streaming __arm_inout("za") { + using Traits = sme_packet_traits<Scalar>; + using Vec = typename Traits::type; + const int svl = Traits::size(); + const svcount_t pc = Traits::ptrue_c(); + const svbool_t all = Traits::ptrue(); + const svbool_t pn = Traits::whilelt(0, cw); + const svbool_t p1lo = Traits::whilelt(0, pw1); + const svbool_t p1hi = Traits::whilelt(svl, pw1); + svzero_za(); + for (Index k = 0; k < depth; ++k) { + const auto a0 = ploadu_x2(pc, blA0 + k * step0); + const Vec a1lo = ploadu(p1lo, blA1 + k * step1); + const Vec a1hi = ploadu(p1hi, sme_offset(blA1, k * step1 + svl)); + const Vec b = ploadu(pn, blB + k * cw); + sme_mopa<0>(all, pn, pget<0>(a0), b); + sme_mopa<1>(all, pn, pget<1>(a0), b); + sme_mopa<2>(p1lo, pn, a1lo, b); + sme_mopa<3>(p1hi, pn, a1hi, b); + } + sme_store_za_tile<Scalar, 0>(C, C_stride_row, C_stride_col, alpha, row_start, svl, col_start, cw); + sme_store_za_tile<Scalar, 1>(C, C_stride_row, C_stride_col, alpha, row_start + svl, svl, col_start, cw); + sme_store_za_tile<Scalar, 2>(C, C_stride_row, C_stride_col, alpha, row_start + 2 * svl, sme_min(pw1, svl), col_start, + cw); + if (pw1 > svl) + sme_store_za_tile<Scalar, 3>(C, C_stride_row, C_stride_col, alpha, row_start + 3 * svl, pw1 - svl, col_start, cw); +} + +// Tile Dst += tile Src, four columns per step. +template <int Dst, int Src, typename Scalar> +EIGEN_ALWAYS_INLINE void sme_fold_tile() __arm_streaming __arm_inout("za") { + const svbool_t all = sme_packet_traits<Scalar>::ptrue(); + for (int c = 0; c < sme_packet_traits<Scalar>::size(); c += 4) { + const auto d = sme_read_ver_za_vg4<Dst, Scalar>(uint32_t(c)), s = sme_read_ver_za_vg4<Src, Scalar>(uint32_t(c)); + sme_write_ver_za_vg4<Dst>(uint32_t(c), + pcreate(padd(all, pget<0>(d), pget<0>(s)), padd(all, pget<1>(d), pget<1>(s)), + padd(all, pget<2>(d), pget<2>(s)), padd(all, pget<3>(d), pget<3>(s)))); + } +} + +// A block that fills one or two tiles (pw or cw <= svl): FMOPAs into one tile wait on each other, so consecutive +// depth steps go to the spare tiles and the partial sums are folded before the store. +template <typename Scalar, typename Index> +EIGEN_ALWAYS_INLINE void sme_process_split(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col, + const Scalar* EIGEN_RESTRICT blA, const Scalar* EIGEN_RESTRICT blB, + Index depth, Scalar alpha, Index row_start, int pw, Index col_start, int cw, + Index a_step) __arm_streaming __arm_inout("za") { + using Traits = sme_packet_traits<Scalar>; + using Vec = typename Traits::type; + const int svl = Traits::size(); + const svbool_t pr0 = Traits::whilelt(0, pw), pr1 = Traits::whilelt(svl, pw); + const svbool_t pc0 = Traits::whilelt(0, cw), pc1 = Traits::whilelt(svl, cw); + svzero_za(); + Index k = 0; + if (pw <= svl && cw <= svl) { + for (; k + 4 <= depth; k += 4) { + const Vec a0 = ploadu(pr0, blA + k * a_step), a1 = ploadu(pr0, blA + (k + 1) * a_step); + const Vec a2 = ploadu(pr0, blA + (k + 2) * a_step), a3 = ploadu(pr0, blA + (k + 3) * a_step); + const Vec b0 = ploadu(pc0, blB + k * cw), b1 = ploadu(pc0, blB + (k + 1) * cw); + const Vec b2 = ploadu(pc0, blB + (k + 2) * cw), b3 = ploadu(pc0, blB + (k + 3) * cw); + sme_mopa<0>(pr0, pc0, a0, b0); + sme_mopa<1>(pr0, pc0, a1, b1); + sme_mopa<2>(pr0, pc0, a2, b2); + sme_mopa<3>(pr0, pc0, a3, b3); + } + for (; k < depth; ++k) sme_mopa<0>(pr0, pc0, ploadu(pr0, blA + k * a_step), ploadu(pc0, blB + k * cw)); + sme_fold_tile<0, 1, Scalar>(); + sme_fold_tile<2, 3, Scalar>(); + sme_fold_tile<0, 2, Scalar>(); + sme_store_za_tile<Scalar, 0>(C, C_stride_row, C_stride_col, alpha, row_start, pw, col_start, cw); + } else if (pw <= svl) { + // Tiles 0 and 1 take the even depth steps, 2 and 3 the odd ones. + for (; k + 2 <= depth; k += 2) { + const Vec a0 = ploadu(pr0, blA + k * a_step), a1 = ploadu(pr0, blA + (k + 1) * a_step); + const Vec b0 = ploadu(pc0, blB + k * cw), b0h = ploadu(pc1, blB + k * cw + svl); + const Vec b1 = ploadu(pc0, blB + (k + 1) * cw), b1h = ploadu(pc1, blB + (k + 1) * cw + svl); + sme_mopa<0>(pr0, pc0, a0, b0); + sme_mopa<1>(pr0, pc1, a0, b0h); + sme_mopa<2>(pr0, pc0, a1, b1); + sme_mopa<3>(pr0, pc1, a1, b1h); + } + if (k < depth) { + const Vec a0 = ploadu(pr0, blA + k * a_step); + sme_mopa<0>(pr0, pc0, a0, ploadu(pc0, blB + k * cw)); + sme_mopa<1>(pr0, pc1, a0, ploadu(pc1, blB + k * cw + svl)); + } + sme_fold_tile<0, 2, Scalar>(); + sme_fold_tile<1, 3, Scalar>(); + sme_store_2x2_grid(C, C_stride_row, C_stride_col, alpha, row_start, pw, 0, col_start, svl, cw - svl); + } else { + // cw <= svl < pw: tiles 0 and 2 take the even depth steps, 1 and 3 the odd ones. + for (; k + 2 <= depth; k += 2) { + const Vec a0 = ploadu(pr0, blA + k * a_step), a0h = ploadu(pr1, blA + k * a_step + svl); + const Vec a1 = ploadu(pr0, blA + (k + 1) * a_step), a1h = ploadu(pr1, blA + (k + 1) * a_step + svl); + const Vec b0 = ploadu(pc0, blB + k * cw), b1 = ploadu(pc0, blB + (k + 1) * cw); + sme_mopa<0>(pr0, pc0, a0, b0); + sme_mopa<2>(pr1, pc0, a0h, b0); + sme_mopa<1>(pr0, pc0, a1, b1); + sme_mopa<3>(pr1, pc0, a1h, b1); + } + if (k < depth) { + const Vec b0 = ploadu(pc0, blB + k * cw); + sme_mopa<0>(pr0, pc0, ploadu(pr0, blA + k * a_step), b0); + sme_mopa<2>(pr1, pc0, ploadu(pr1, blA + k * a_step + svl), b0); + } + sme_fold_tile<0, 1, Scalar>(); + sme_fold_tile<2, 3, Scalar>(); + sme_store_2x2_grid(C, C_stride_row, C_stride_col, alpha, row_start, svl, pw - svl, col_start, cw, 0); + } +} + +// One pw x cw block: sme_process_split when it fills at most two tiles of the 2 x 2 grid; complex blocks keep +// sme_process. `split_ok` (sme_split_ok) holds once per call, outside the block loops. +template <bool ConjLhs, bool ConjRhs, bool FoldRealAlpha, typename Scalar, typename Index> +EIGEN_ALWAYS_INLINE void sme_process_block(Scalar* C, Index rs, Index cs, const Scalar* blA, const Scalar* blB, + Index depth, Scalar alpha, Index row_start, int pw, Index col_start, int cw, + Index a_step, bool split_ok, Index, + Index) __arm_streaming __arm_inout("za") { + const int svl = sme_packet_traits<Scalar>::size(); + if (split_ok && (pw <= svl || cw <= svl)) + sme_process_split(C, rs, cs, blA, blB, depth, alpha, row_start, pw, col_start, cw, a_step); + else + sme_process<ConjLhs, ConjRhs>(C, rs, cs, blA, blB, depth, alpha, row_start, pw, col_start, cw, a_step); +} +template <bool ConjLhs, bool ConjRhs, bool FoldRealAlpha, typename RealScalar, typename Index> +EIGEN_ALWAYS_INLINE void sme_process_block(std::complex<RealScalar>* C, Index rs, Index cs, + const std::complex<RealScalar>* blA, const std::complex<RealScalar>* blB, + Index depth, std::complex<RealScalar> alpha, Index row_start, int pw, + Index col_start, int cw, Index a_step, bool, Index rows, + Index cols) __arm_streaming __arm_inout("za") { + sme_process<ConjLhs, ConjRhs, FoldRealAlpha>(C, rs, cs, blA, blB, depth, alpha, row_start, pw, col_start, cw, a_step, + rows, cols); +} + +// Whether the vector length gives the tile shapes the block kernels assume: the depth-split kernel covers one 2 x 2 +// grid of blocks up to MR x NR with four-slice tile folds, and the narrow kernel stacks two LHS panels of 2*svl rows. +template <typename Scalar> +EIGEN_ALWAYS_INLINE bool sme_split_ok() __arm_streaming { + const int svl = sme_packet_traits<typename NumTraits<Scalar>::Real>::size(); + return svl >= 4 && sme_block<Scalar>::mr <= 2 * svl && sme_block<Scalar>::nr <= 2 * svl; +} +template <typename Scalar> +EIGEN_ALWAYS_INLINE bool sme_narrow_ok() __arm_streaming { + return sme_block<Scalar>::mr == 2 * sme_packet_traits<typename NumTraits<Scalar>::Real>::size(); +} + +// sme_process_narrow for real scalars; complex panels keep the 2 x 2 path. +template <typename Scalar, typename Index> +EIGEN_ALWAYS_INLINE void sme_process_narrow_dispatch(Scalar* C, Index rs, Index cs, const Scalar* blA0, Index step0, + const Scalar* blA1, Index step1, int pw1, const Scalar* blB, + int cw, Index depth, Scalar alpha, Index row_start, + Index col_start) __arm_streaming __arm_inout("za") { + sme_process_narrow(C, rs, cs, blA0, step0, blA1, step1, pw1, blB, cw, depth, alpha, row_start, col_start); +} +template <typename RealScalar, typename Index> +EIGEN_ALWAYS_INLINE void sme_process_narrow_dispatch(std::complex<RealScalar>*, Index, Index, + const std::complex<RealScalar>*, Index, + const std::complex<RealScalar>*, Index, int, + const std::complex<RealScalar>*, int, Index, + std::complex<RealScalar>, Index, + Index) __arm_streaming __arm_inout("za") {} + +// Core-side prefetch into L2 of the column-major C block the next sme_process call reads and writes. +template <typename Scalar, typename Index> +static EIGEN_ALWAYS_INLINE void sme_prefetch_next_c(const Scalar* C, Index C_stride_row, Index C_stride_col, Index i, + Index j, Index rows, Index cols, int mr, + int nr) __arm_streaming_compatible { + if (C_stride_row != 1 || j >= cols) return; + const Index h = sme_min(rows - i, Index(mr)), w = sme_min(cols - j, Index(nr)); + const Index bytes = h * Index(sizeof(Scalar)); + for (Index c = 0; c < w; ++c) + for (Index b = 0; b < bytes; b += 128) + __builtin_prefetch(reinterpret_cast<const char*>(C + i + (j + c) * C_stride_col) + b, 1, 2); +} + +template <typename Scalar, bool ConjLhs, bool ConjRhs, bool FoldRealAlpha = false, typename Index> EIGEN_DONT_INLINE __arm_locally_streaming __arm_new("za") void sme_gebp_impl( Scalar* C, Index C_stride_row, Index C_stride_col, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { @@ -1860,20 +2142,128 @@ // GeneralMatrixMatrix.h ensures blockA fits in L2 via mc-blocking. Each // packed panel is depth-major with depth-stride equal to its width (MR/NR // for full panels, the tail width otherwise), so that width is passed as - // both the logical block size and the load stride to sme_process; partial - // blocks are tiled and predicated inside the generic path. + // both the logical block size and the load stride to the block kernels. + // The narrow kernel stacks two LHS panels of exactly 2*svl rows. + // A small C block (256 KB or less) gains nothing from the prefetch and pays its instructions on every call. + const bool prefetch_c = rows * cols * Index(sizeof(Scalar)) > Index(256 * 1024); + const bool split_ok = sme_split_ok<Scalar>(), narrow_ok = sme_narrow_ok<Scalar>(); for (Index j = 0; j < cols; j += NR) { const int cw = static_cast<int>(sme_min(cols - j, Index(NR))); const Scalar* blB = blockB + j * strideB + offsetB * cw; - for (Index i = 0; i < rows; i += MR) { + Index i = 0; + EIGEN_IF_CONSTEXPR (!NumTraits<Scalar>::IsComplex) { + if (narrow_ok && cw <= sme_packet_traits<typename NumTraits<Scalar>::Real>::size()) + for (; i + MR < rows; i += 2 * MR) { + const int pw1 = static_cast<int>(sme_min(rows - i - MR, Index(MR))); + sme_process_narrow_dispatch(C, C_stride_row, C_stride_col, blockA + i * strideA + offsetA * MR, Index(MR), + blockA + (i + MR) * strideA + offsetA * pw1, Index(pw1), pw1, blB, cw, depth, + alpha, i, j); + } + } + for (; i < rows; i += MR) { const int pw = static_cast<int>(sme_min(rows - i, Index(MR))); const Scalar* blA = blockA + i * strideA + offsetA * pw; - sme_process<ConjLhs, ConjRhs>(C, C_stride_row, C_stride_col, blA, blB, depth, alpha, i, pw, j, cw); + if (prefetch_c) + sme_prefetch_next_c(C, C_stride_row, C_stride_col, i + MR < rows ? i + MR : Index(0), + i + MR < rows ? j : j + NR, rows, cols, MR, NR); + sme_process_block<ConjLhs, ConjRhs, FoldRealAlpha>(C, C_stride_row, C_stride_col, blA, blB, depth, alpha, i, pw, + j, cw, Index(pw), split_ok, rows, cols); } } } +// Picks the sme_gebp_impl instantiation. A complex product with a real alpha +// other than 1, e.g. the -1 of C.noalias() -= A*B, gets the kernel that folds +// C into one FMA per component (see sme_accumulate_pair_real_alpha); every +// other product gets the kernel without that path. Real scalars have one. +template <typename Scalar, bool ConjLhs, bool ConjRhs, bool IsComplex = NumTraits<Scalar>::IsComplex> +struct sme_gebp_dispatch { + template <typename Index> + static void run(Scalar* C, Index C_stride_row, Index C_stride_col, const Scalar* blockA, const Scalar* blockB, + Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, + Index offsetB) { + sme_gebp_impl<Scalar, ConjLhs, ConjRhs, false>(C, C_stride_row, C_stride_col, blockA, blockB, rows, depth, cols, + alpha, strideA, strideB, offsetA, offsetB); + } +}; + +template <typename Scalar, bool ConjLhs, bool ConjRhs> +struct sme_gebp_dispatch<Scalar, ConjLhs, ConjRhs, true> { + template <typename Index> + static void run(Scalar* C, Index C_stride_row, Index C_stride_col, const Scalar* blockA, const Scalar* blockB, + Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, + Index offsetB) { + using RealScalar = typename NumTraits<Scalar>::Real; + if (numext::imag(alpha) == RealScalar(0) && numext::real(alpha) != RealScalar(1)) { + sme_gebp_impl<Scalar, ConjLhs, ConjRhs, true>(C, C_stride_row, C_stride_col, blockA, blockB, rows, depth, cols, + alpha, strideA, strideB, offsetA, offsetB); + } else { + sme_gebp_impl<Scalar, ConjLhs, ConjRhs, false>(C, C_stride_row, C_stride_col, blockA, blockB, rows, depth, cols, + alpha, strideA, strideB, offsetA, offsetB); + } + } +}; + +// gebp with the LHS read from a ColMajor source: rows i..i+pw of column k are +// at lhs + i + k * lda, the packed layout with a_step = lda. Real scalars only. +template <typename Scalar, typename Index> +EIGEN_DONT_INLINE __arm_locally_streaming __arm_new("za") void sme_gebp_impl_direct_lhs( + Scalar* C, Index C_stride_row, Index C_stride_col, const Scalar* lhs, Index lda, const Scalar* blockB, Index rows, + Index depth, Index cols, Scalar alpha, Index strideB, Index offsetB) { + constexpr int MR = sme_block<Scalar>::mr; + constexpr int NR = sme_block<Scalar>::nr; + const bool split_ok = sme_split_ok<Scalar>(), narrow_ok = sme_narrow_ok<Scalar>(); + for (Index j = 0; j < cols; j += NR) { + const int cw = static_cast<int>(sme_min(cols - j, Index(NR))); + const Scalar* blB = blockB + j * strideB + offsetB * cw; + Index i = 0; + if (narrow_ok && cw <= sme_packet_traits<Scalar>::size()) + for (; i + MR < rows; i += 2 * MR) + sme_process_narrow(C, C_stride_row, C_stride_col, lhs + i, lda, lhs + i + MR, lda, + static_cast<int>(sme_min(rows - i - MR, Index(MR))), blB, cw, depth, alpha, i, j); + for (; i < rows; i += MR) { + const int pw = static_cast<int>(sme_min(rows - i, Index(MR))); + sme_process_block<false, false, false>(C, C_stride_row, C_stride_col, lhs + i, blB, depth, alpha, i, pw, j, cw, + lda, split_ok, rows, cols); + } + } +} + +// In-place ColMajor LHS: deep enough for the ZA kernel, a stride that is not a 4 KB multiple (such columns map to one +// L1 set) and a block of at most 4 MB, past which the strided re-reads lose to the packed panel. +#ifndef EIGEN_SME_DIRECT_LHS_MAX_STRIDE_BYTES +#define EIGEN_SME_DIRECT_LHS_MAX_STRIDE_BYTES 16384 +#endif +#ifndef EIGEN_SME_DIRECT_LHS_MAX_BLOCK_BYTES +#define EIGEN_SME_DIRECT_LHS_MAX_BLOCK_BYTES (4 << 20) +#endif +#ifndef EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES +#define EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES (16 << 20) +#endif +template <typename Scalar, typename Index> +bool sme_direct_lhs_ok(Index lhsStride, Index rows, Index depth, Index cols) { +#ifdef EIGEN_SME_FORCE_NEON_SMALL_BLOCKS + EIGEN_UNUSED_VARIABLE(lhsStride); + EIGEN_UNUSED_VARIABLE(rows); + EIGEN_UNUSED_VARIABLE(depth); + EIGEN_UNUSED_VARIABLE(cols); + return false; +#else + if (NumTraits<Scalar>::IsComplex || depth <= Index(sme_neon_max_depth<Scalar>::value)) return false; + // A RHS of one panel reads each LHS element once, so packing it only adds a copy: always for the narrow kernel, + // and for the 2 x 2 one while a panel spans few enough pages. + const std::size_t stride_bytes = std::size_t(lhsStride) * sizeof(Scalar); + if (cols <= Index(sme_block<Scalar>::nr)) { + const std::size_t span_limit = std::size_t(EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES); + if (span_limit == 0) return false; + return cols <= Index(sme_block<Scalar>::nr / 2) || std::size_t(depth) * stride_bytes <= span_limit; + } + return stride_bytes % 4096 != 0 && stride_bytes <= std::size_t(EIGEN_SME_DIRECT_LHS_MAX_STRIDE_BYTES) && + std::size_t(rows) * std::size_t(depth) * sizeof(Scalar) <= std::size_t(EIGEN_SME_DIRECT_LHS_MAX_BLOCK_BYTES); +#endif +} + // NEON path for small blocks: a shallow block pays the ZA enable and a dependent FMOPA chain per // tile for little work, so its panels are consumed with NEON in the SME layout (one depth step is // pw contiguous scalars, complex ones as pw reals then pw imaginaries). @@ -2300,8 +2690,24 @@ return; } - sme_gebp_impl<Scalar, ConjugateLhs, ConjugateRhs>(C_base, C_stride_row, C_stride_col, blockA, blockB, rows, depth, - cols, alpha, strideA, strideB, offsetA, offsetB); + sme_fpsr_guard fpsr; + sme_gebp_dispatch<Scalar, ConjugateLhs, ConjugateRhs>::run(C_base, C_stride_row, C_stride_col, blockA, blockB, rows, + depth, cols, alpha, strideA, strideB, offsetA, offsetB); + } + + // The LHS block read from its ColMajor source (see sme_direct_lhs_ok); only + // instantiated for real scalars. + EIGEN_DONT_INLINE void run_direct_lhs(const DataMapper& res, const Scalar* lhs, Index lhsStride, const Scalar* blockB, + Index rows, Index depth, Index cols, ResScalar alpha, Index strideB = -1, + Index offsetB = 0) { + if (strideB == -1) strideB = depth; + if (rows <= 0 || cols <= 0 || depth <= 0) return; + Scalar* C_base = const_cast<Scalar*>(&res(0, 0)); + const Index C_stride_row = &res(1, 0) - &res(0, 0); + const Index C_stride_col = &res(0, 1) - &res(0, 0); + sme_fpsr_guard fpsr; + sme_gebp_impl_direct_lhs<Scalar, Index>(C_base, C_stride_row, C_stride_col, lhs, lhsStride, blockB, rows, depth, + cols, alpha, strideB, offsetB); } }; @@ -2482,7 +2888,10 @@ template <typename Scalar, int StorageOrder, bool IsLhs, typename Index> EIGEN_DONT_INLINE void sme_symm_pack_panels(Scalar* block, const Scalar* EIGEN_RESTRICT base, Index stride, Index depth, Index outer, Index k2) { - sme_symm_pack_dense_regions<Scalar, StorageOrder, IsLhs, Index>(block, base, stride, depth, outer, k2); + { + sme_fpsr_guard fpsr; + sme_symm_pack_dense_regions<Scalar, StorageOrder, IsLhs, Index>(block, base, stride, depth, outer, k2); + } sme_symm_pack_straddle<Scalar, StorageOrder, IsLhs, Index>(block, base, stride, depth, outer, k2); } @@ -2523,6 +2932,241 @@ #undef EIGEN_SME_DECLARE_SYMM_PACKERS +// Tiny results (at most 2 * PS x 8): a NEON outer-product kernel with the whole result in registers, called before +// any blocking or packing. Both the coeff-based product (one dependent FMA chain per result) and the packed paths +// are several times slower there. +template <typename Scalar> +struct sme_tiny_neon; +template <> +struct sme_tiny_neon<float> { + using V = float32x4_t; + static constexpr int PS = 4; + static EIGEN_ALWAYS_INLINE V ld(const float* p) { return vld1q_f32(p); } + static EIGEN_ALWAYS_INLINE V zero() { return vdupq_n_f32(0.f); } + // The first n (< PS) scalars at p, zeros after them. + static EIGEN_ALWAYS_INLINE V ld_part(const float* p, int n) { + if (n <= 0) return zero(); + if (n == 1) return vld1q_lane_f32(p, zero(), 0); + const V v = vcombine_f32(vld1_f32(p), vdup_n_f32(0.f)); + return n == 2 ? v : vld1q_lane_f32(p + 2, v, 2); + } + // A full load permuted by byte indices; indices of 16 or more give zeros. + static EIGEN_ALWAYS_INLINE V ld_tbl(const float* p, uint8x16_t idx) { + return vreinterpretq_f32_u8(vqtbl1q_u8(vreinterpretq_u8_f32(vld1q_f32(p)), idx)); + } + template <int L> + static EIGEN_ALWAYS_INLINE V fma_lane(V c, V a, V b) { + return vfmaq_laneq_f32(c, a, b, L); + } + static EIGEN_ALWAYS_INLINE V fma_n(V c, V a, float b) { return vfmaq_n_f32(c, a, b); } + static EIGEN_ALWAYS_INLINE void st(float* p, V v) { vst1q_f32(p, v); } + static EIGEN_ALWAYS_INLINE V add(V a, V b) { return vaddq_f32(a, b); } + // Rows in, columns out. + static EIGEN_ALWAYS_INLINE void transpose(V (&r)[PS]) { + const float64x2_t t0 = vreinterpretq_f64_f32(vtrn1q_f32(r[0], r[1])); + const float64x2_t t1 = vreinterpretq_f64_f32(vtrn2q_f32(r[0], r[1])); + const float64x2_t t2 = vreinterpretq_f64_f32(vtrn1q_f32(r[2], r[3])); + const float64x2_t t3 = vreinterpretq_f64_f32(vtrn2q_f32(r[2], r[3])); + r[0] = vreinterpretq_f32_f64(vtrn1q_f64(t0, t2)); + r[1] = vreinterpretq_f32_f64(vtrn1q_f64(t1, t3)); + r[2] = vreinterpretq_f32_f64(vtrn2q_f64(t0, t2)); + r[3] = vreinterpretq_f32_f64(vtrn2q_f64(t1, t3)); + } +}; +template <> +struct sme_tiny_neon<double> { + using V = float64x2_t; + static constexpr int PS = 2; + static EIGEN_ALWAYS_INLINE V ld(const double* p) { return vld1q_f64(p); } + static EIGEN_ALWAYS_INLINE V zero() { return vdupq_n_f64(0.); } + static EIGEN_ALWAYS_INLINE V ld_part(const double* p, int n) { return n > 0 ? vld1q_lane_f64(p, zero(), 0) : zero(); } + static EIGEN_ALWAYS_INLINE V ld_tbl(const double* p, uint8x16_t idx) { + return vreinterpretq_f64_u8(vqtbl1q_u8(vreinterpretq_u8_f64(vld1q_f64(p)), idx)); + } + template <int L> + static EIGEN_ALWAYS_INLINE V fma_lane(V c, V a, V b) { + return vfmaq_laneq_f64(c, a, b, L); + } + static EIGEN_ALWAYS_INLINE V fma_n(V c, V a, double b) { return vfmaq_n_f64(c, a, b); } + static EIGEN_ALWAYS_INLINE void st(double* p, V v) { vst1q_f64(p, v); } + static EIGEN_ALWAYS_INLINE V add(V a, V b) { return vaddq_f64(a, b); } + static EIGEN_ALWAYS_INLINE void transpose(V (&r)[PS]) { + const V t0 = vtrn1q_f64(r[0], r[1]); + r[1] = vtrn2q_f64(r[0], r[1]); + r[0] = t0; + } +}; + +// One depth chunk of FMAs; the lane index must be a constant, hence the recursion over it. Depth step u of a chunk +// accumulates into set u % S: four FMA pipes of latency four need about 16 independent chains. +template <typename T, int RV, int NC, int S> +struct sme_tiny_step { + using V = typename T::V; + static constexpr int PS = T::PS; + // ColMajor B: b[j] holds depth steps k .. k + PS - 1 of column j; lane U is step k + U. + template <int U> + static EIGEN_ALWAYS_INLINE void col(V (&acc)[S][NC][RV], const V (&a)[PS][RV], const V (&b)[NC], + std::integral_constant<int, U>) { + EIGEN_UNROLL_LOOP + for (int j = 0; j < NC; ++j) { + EIGEN_UNROLL_LOOP + for (int r = 0; r < RV; ++r) acc[U % S][j][r] = T::template fma_lane<U>(acc[U % S][j][r], a[U][r], b[j]); + } + col(acc, a, b, std::integral_constant<int, U + 1>()); + } + static EIGEN_ALWAYS_INLINE void col(V (&)[S][NC][RV], const V (&)[PS][RV], const V (&)[NC], + std::integral_constant<int, PS>) {} + // RowMajor B: b[c] holds columns c * PS .. of one depth step. + template <int J> + static EIGEN_ALWAYS_INLINE void row(V (&acc)[NC][RV], const V (&a)[RV], const V (&b)[NC / PS], + std::integral_constant<int, J>) { + EIGEN_UNROLL_LOOP + for (int r = 0; r < RV; ++r) acc[J][r] = T::template fma_lane<J % PS>(acc[J][r], a[r], b[J / PS]); + row(acc, a, b, std::integral_constant<int, J + 1>()); + } + static EIGEN_ALWAYS_INLINE void row(V (&)[NC][RV], const V (&)[RV], const V (&)[NC / PS], + std::integral_constant<int, NC>) {} +}; + +// C (m x n, m <= RV * PS, n <= NC) += alpha * A * B, reading only A's m rows and B's n columns: rows and columns +// past the result repeat the last valid one or load as zeros, and their sums are dropped. +template <typename Scalar, int RV, int NC, int LhsOrder, int RhsOrder, typename Index> +EIGEN_DONT_INLINE void sme_tiny_gemm_kernel(Index m, Index n, Index depth, const Scalar* A, Index lda, const Scalar* B, + Index ldb, Scalar* C, Index incr, Index ldc, Scalar alpha) { + using T = sme_tiny_neon<Scalar>; + using V = typename T::V; + constexpr int PS = T::PS, MR = RV * PS, S = 16 / (RV * NC) < PS ? 16 / (RV * NC) : PS; + using Step = sme_tiny_step<T, RV, NC, S>; + V acc[S][NC][RV]; + EIGEN_UNROLL_LOOP + for (int t = 0; t < S; ++t) { + EIGEN_UNROLL_LOOP + for (int j = 0; j < NC; ++j) { + EIGEN_UNROLL_LOOP + for (int r = 0; r < RV; ++r) acc[t][j][r] = T::zero(); + } + } + const Scalar* a_row[MR]; + for (int i = 0; i < MR; ++i) a_row[i] = A + numext::mini(Index(i), m - 1) * lda; + const Scalar* b_col[NC]; + for (int j = 0; j < NC; ++j) b_col[j] = B + numext::mini(Index(j), n - 1) * ldb; + // A column (B row) vector r holds cnt valid scalars. Once m (n) >= PS, each is one full load ending at its last + // valid scalar, shifted down by a byte table with zeros after; below that, a partial load. + int a_cnt[RV], b_cnt[NC / PS]; + Index a_off[RV], b_off[NC / PS]; + uint8x16_t a_idx[RV], b_idx[NC / PS]; + const auto shift_table = [](int cnt) { + EIGEN_ALIGN16 uint8_t t[16]; + for (int b = 0; b < 16; ++b) + t[b] = b < cnt * int(sizeof(Scalar)) ? uint8_t(b + (PS - cnt) * int(sizeof(Scalar))) : 0xff; + return vld1q_u8(t); + }; + for (int r = 0; r < RV; ++r) { + a_cnt[r] = int(numext::mini(Index(PS), m - r * PS)); + a_off[r] = r * PS + a_cnt[r] - PS; + a_idx[r] = shift_table(a_cnt[r]); + } + for (int c = 0; c < NC / PS; ++c) { + b_cnt[c] = int(numext::maxi(Index(0), numext::mini(Index(PS), n - c * PS))); + b_off[c] = numext::maxi(Index(0), Index(c * PS + b_cnt[c] - PS)); + b_idx[c] = shift_table(b_cnt[c]); + } + const bool a_full = m >= PS, b_full = n >= PS; + const Index kmain = (depth / PS) * PS; + for (Index k = 0; k < kmain; k += PS) { + V a[PS][RV]; + EIGEN_IF_CONSTEXPR (LhsOrder == ColMajor) { + EIGEN_UNROLL_LOOP + for (int u = 0; u < PS; ++u) { + EIGEN_UNROLL_LOOP + for (int r = 0; r < RV; ++r) + a[u][r] = a_cnt[r] == PS ? T::ld(A + (k + u) * lda + r * PS) + : a_full ? T::ld_tbl(A + (k + u) * lda + a_off[r], a_idx[r]) + : T::ld_part(A + (k + u) * lda + r * PS, a_cnt[r]); + } + } else { + EIGEN_UNROLL_LOOP + for (int r = 0; r < RV; ++r) { + V blk[PS]; + EIGEN_UNROLL_LOOP + for (int q = 0; q < PS; ++q) blk[q] = T::ld(a_row[r * PS + q] + k); + T::transpose(blk); + EIGEN_UNROLL_LOOP + for (int u = 0; u < PS; ++u) a[u][r] = blk[u]; + } + } + EIGEN_IF_CONSTEXPR (RhsOrder == ColMajor) { + V b[NC]; + EIGEN_UNROLL_LOOP + for (int j = 0; j < NC; ++j) b[j] = T::ld(b_col[j] + k); + Step::col(acc, a, b, std::integral_constant<int, 0>()); + } else { + EIGEN_UNROLL_LOOP + for (int u = 0; u < PS; ++u) { + V b[NC / PS]; + EIGEN_UNROLL_LOOP + for (int c = 0; c < NC / PS; ++c) + b[c] = b_cnt[c] == PS ? T::ld(B + (k + u) * ldb + c * PS) + : b_cnt[c] == 0 ? T::zero() + : b_full ? T::ld_tbl(B + (k + u) * ldb + b_off[c], b_idx[c]) + : T::ld_part(B + (k + u) * ldb + c * PS, b_cnt[c]); + Step::row(acc[u % S], a[u], b, std::integral_constant<int, 0>()); + } + } + } + for (Index k = kmain; k < depth; ++k) { + EIGEN_ALIGN16 Scalar at[MR]; + for (int i = 0; i < MR; ++i) + at[i] = LhsOrder == ColMajor ? A[k * lda + numext::mini(Index(i), m - 1)] : a_row[i][k]; + V a[RV]; + EIGEN_UNROLL_LOOP + for (int r = 0; r < RV; ++r) a[r] = T::ld(at + r * PS); + EIGEN_UNROLL_LOOP + for (int j = 0; j < NC; ++j) { + const Scalar b = RhsOrder == ColMajor ? b_col[j][k] : B[k * ldb + numext::mini(Index(j), n - 1)]; + EIGEN_UNROLL_LOOP + for (int r = 0; r < RV; ++r) acc[0][j][r] = T::fma_n(acc[0][j][r], a[r], b); + } + } + EIGEN_UNROLL_LOOP + for (int t = 1; t < S; ++t) { + EIGEN_UNROLL_LOOP + for (int j = 0; j < NC; ++j) { + EIGEN_UNROLL_LOOP + for (int r = 0; r < RV; ++r) acc[0][j][r] = T::add(acc[0][j][r], acc[t][j][r]); + } + } + EIGEN_ALIGN16 Scalar out[NC][MR]; + EIGEN_UNROLL_LOOP + for (int j = 0; j < NC; ++j) { + EIGEN_UNROLL_LOOP + for (int r = 0; r < RV; ++r) T::st(&out[j][r * PS], acc[0][j][r]); + } + for (Index j = 0; j < n; ++j) + for (Index i = 0; i < m; ++i) C[i * incr + j * ldc] += alpha * out[j][i]; +} + +template <typename Scalar, int LhsOrder, int RhsOrder, typename Index> +bool sme_tiny_gemm(Index rows, Index cols, Index depth, const Scalar* lhs, Index lhsStride, const Scalar* rhs, + Index rhsStride, Scalar* res, Index resIncr, Index resStride, Scalar alpha) { + constexpr int PS = sme_tiny_neon<Scalar>::PS; + if (!sme_tiny_gemm_wins<Scalar>(rows, cols, depth)) return false; + const bool one_vec = rows <= PS, four_cols = cols <= 4; + if (one_vec && four_cols) + sme_tiny_gemm_kernel<Scalar, 1, 4, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, + resIncr, resStride, alpha); + else if (one_vec) + sme_tiny_gemm_kernel<Scalar, 1, 8, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, + resIncr, resStride, alpha); + else if (four_cols) + sme_tiny_gemm_kernel<Scalar, 2, 4, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, + resIncr, resStride, alpha); + else + sme_tiny_gemm_kernel<Scalar, 2, 8, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, + resIncr, resStride, alpha); + return true; +} + } // namespace internal } // namespace Eigen
diff --git a/Eigen/src/Core/arch/SME/PacketMath.h b/Eigen/src/Core/arch/SME/PacketMath.h new file mode 100644 index 0000000..c6e9e3b --- /dev/null +++ b/Eigen/src/Core/arch/SME/PacketMath.h
@@ -0,0 +1,451 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#ifndef EIGEN_PACKET_MATH_SME_H +#define EIGEN_PACKET_MATH_SME_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { +namespace internal { + +// --------------------------------------------------------------------------- +// Streaming-mode packet layer of the SME backend. +// +// The evaluators keep NEON packets; the streaming SVE registers and the ZA array +// are reached only from the product kernels, through the operations below, which +// cannot be the packet_traits type or p* specializations: +// - a streaming vector (svfloat32_t, ...) is sizeless -- no sizeof, not a +// member or array element type, lane count known at run time -- so neither +// unpacket_traits<>::size nor PacketBlock can describe it; +// - code touching one must carry __arm_streaming (or _compatible), which is +// part of the function type: a p* specialization cannot add it, and an +// evaluator cannot call it without a mode switch, nor at all where the +// target has no non-streaming SVE (-mcpu=apple-m4); +// - streaming mode runs scalar FP 30-80x slower and NEON not at all on Apple M4. +// So these are __arm_streaming overloads on the streaming vector types: Eigen's +// names and argument order where a counterpart exists (pset1, ploadu, pstoreu, +// padd, pmul, pnegate, pmadd, pnmadd, predux), the arithmetic and memory +// operations taking the SVE governing predicate first; the SVE name with Eigen's +// p prefix where none does (pget, pcreate, pld2, pst2, puzp1/2, pzip1/2, psplice); an sme_ +// prefix for the ZA tile operations; and the predicates and lane counts as +// members of sme_packet_traits. Eigen's templates of the same names drop out of +// overload resolution, by arity or by substitution failure on the empty +// unpacket_traits below. +// --------------------------------------------------------------------------- + +// ZA tile count at a given element width: 4 ZA.S tiles, 8 ZA.D tiles. Architectural rather than +// feature-dependent, so it stays outside sme_packet_traits, which has no double specialization +// without FEAT_SME_F64F64. +template <typename RealScalar> +struct sme_tile_count; +template <> +struct sme_tile_count<float> { + static constexpr int value = 4; +}; +template <> +struct sme_tile_count<double> { + static constexpr int value = 8; +}; + +// Scalar -> streaming vector, its two- and four-vector tuples, and the predicates of its element +// width; size() is the lane count of one streaming vector, a runtime value. whilelt takes int64_t: +// svwhilelt_b* is overloaded on the four fixed-width types only, so Index -- `long` where int64_t +// is `long long` -- matches none of them exactly; every bound the kernels pass is a non-negative +// Index. +template <typename Scalar> +struct sme_packet_traits {}; + +template <> +struct sme_packet_traits<float> { + using type = svfloat32_t; + using type_x2 = svfloat32x2_t; + using type_x4 = svfloat32x4_t; + static EIGEN_ALWAYS_INLINE int size() __arm_streaming_compatible { return static_cast<int>(svcntsw()); } + static EIGEN_ALWAYS_INLINE svbool_t ptrue() __arm_streaming { return svptrue_b32(); } + static EIGEN_ALWAYS_INLINE svcount_t ptrue_c() __arm_streaming { return svptrue_c32(); } + static EIGEN_ALWAYS_INLINE svbool_t whilelt(int64_t begin, int64_t end) __arm_streaming { + return svwhilelt_b32(begin, end); + } + static EIGEN_ALWAYS_INLINE svcount_t whilelt_c4(int64_t begin, int64_t end) __arm_streaming { + return svwhilelt_c32_s64(begin, end, 4); + } +}; + +#ifdef EIGEN_VECTORIZE_SME_F64F64 +template <> +struct sme_packet_traits<double> { + using type = svfloat64_t; + using type_x2 = svfloat64x2_t; + using type_x4 = svfloat64x4_t; + static EIGEN_ALWAYS_INLINE int size() __arm_streaming_compatible { return static_cast<int>(svcntsd()); } + static EIGEN_ALWAYS_INLINE svbool_t ptrue() __arm_streaming { return svptrue_b64(); } + static EIGEN_ALWAYS_INLINE svcount_t ptrue_c() __arm_streaming { return svptrue_c64(); } + static EIGEN_ALWAYS_INLINE svbool_t whilelt(int64_t begin, int64_t end) __arm_streaming { + return svwhilelt_b64(begin, end); + } + static EIGEN_ALWAYS_INLINE svcount_t whilelt_c4(int64_t begin, int64_t end) __arm_streaming { + return svwhilelt_c64_s64(begin, end, 4); + } +}; +#endif + +// Streaming vector -> scalar. The primary template has no members, so pset1's parameter type +// below fails substitution for every other packet type and Eigen's own pset1 is selected there. +template <typename Packet> +struct sme_unpacket_traits {}; +template <> +struct sme_unpacket_traits<svfloat32_t> { + using type = float; + static EIGEN_ALWAYS_INLINE svfloat32_t dup(float from) __arm_streaming { return svdup_f32(from); } +}; +#ifdef EIGEN_VECTORIZE_SME_F64F64 +template <> +struct sme_unpacket_traits<svfloat64_t> { + using type = double; + static EIGEN_ALWAYS_INLINE svfloat64_t dup(double from) __arm_streaming { return svdup_f64(from); } +}; +#endif + +// A streaming vector is neither a packet nor a scalar, so unpacket_traits has no members for it: a +// signature naming unpacket_traits<Packet>::type, such as Eigen's pset1, fails substitution, and a +// generic operation called without its predicate (predux(v), pfirst(v)) does not compile, where +// mapping the vector to itself would make it a one-lane "scalar" and resolve those to the identity. +// The primary template cannot serve: it applies sizeof, which a sizeless type does not have. +template <> +struct unpacket_traits<svfloat32_t> {}; +#ifdef EIGEN_VECTORIZE_SME_F64F64 +template <> +struct unpacket_traits<svfloat64_t> {}; +#endif + +// An overload of Eigen's pset1 rather than a specialization (see the file comment); the two never +// compete, since each fails substitution on the other's vector types. +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet pset1(typename sme_unpacket_traits<Packet>::type from) __arm_streaming { + return sme_unpacket_traits<Packet>::dup(from); +} + +// Every operation takes its governing predicate, as the ACLE intrinsics do: the kernels keep the +// predicate of the enclosing loop live, and an all-true one materialized per operation instead +// costs a PTRUE and a P register inside the ZA store loops (48 PTRUEs and 10 predicate spills +// against 10 and none in the float GEMM kernel). The vector type follows the scalar pointer or the +// vector arguments, so one template per operation covers every element width through the +// type-generic ACLE overloads. SVE loads have no alignment requirement, hence only the unaligned +// spellings; _x2/_x4 move two and four consecutive vectors under one SME2 predicate-as-counter. +template <typename Scalar> +EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type ploadu(svbool_t pg, const Scalar* from) __arm_streaming { + return svld1(pg, from); +} +template <typename Scalar> +EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type_x2 ploadu_x2(svcount_t pn, + const Scalar* from) __arm_streaming { + return svld1_x2(pn, from); +} +template <typename Scalar> +EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type_x4 ploadu_x4(svcount_t pn, + const Scalar* from) __arm_streaming { + return svld1_x4(pn, from); +} +template <typename Scalar> +EIGEN_ALWAYS_INLINE void pstoreu(svbool_t pg, Scalar* to, + typename sme_packet_traits<Scalar>::type from) __arm_streaming { + svst1(pg, to, from); +} +template <typename Scalar> +EIGEN_ALWAYS_INLINE void pstoreu_x2(svcount_t pn, Scalar* to, + typename sme_packet_traits<Scalar>::type_x2 from) __arm_streaming { + svst1(pn, to, from); +} +template <typename Scalar> +EIGEN_ALWAYS_INLINE void pstoreu_x4(svcount_t pn, Scalar* to, + typename sme_packet_traits<Scalar>::type_x4 from) __arm_streaming { + svst1(pn, to, from); +} +// Two-element structure load and store (LD2/ST2): pld2 splits interleaved pairs into the even and odd +// lanes, the real and imaginary parts of a complex array, and pst2 interleaves them back. +template <typename Scalar> +EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type_x2 pld2(svbool_t pg, const Scalar* from) __arm_streaming { + return svld2(pg, from); +} +template <typename Scalar> +EIGEN_ALWAYS_INLINE void pst2(svbool_t pg, Scalar* to, + typename sme_packet_traits<Scalar>::type_x2 from) __arm_streaming { + svst2(pg, to, from); +} + +// Arithmetic in Eigen's argument order after the predicate: pmadd(pg, a, b, c) = a * b + c and +// pnmadd(pg, a, b, c) = c - a * b, each one fused operation. Inactive lanes are unspecified (the +// _x forms); every consumer stores through a predicate of its own. +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet padd(svbool_t pg, Packet a, Packet b) __arm_streaming { + return svadd_x(pg, a, b); +} +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet pmul(svbool_t pg, Packet a, Packet b) __arm_streaming { + return svmul_x(pg, a, b); +} +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet pnegate(svbool_t pg, Packet a) __arm_streaming { + return svneg_x(pg, a); +} +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet pmadd(svbool_t pg, Packet a, Packet b, Packet c) __arm_streaming { + return svmla_x(pg, c, a, b); +} +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet pnmadd(svbool_t pg, Packet a, Packet b, Packet c) __arm_streaming { + return svmls_x(pg, c, a, b); +} +// Merging form: inactive lanes keep c, in the one FMLA that a select after the _x form does not fold to. +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet pmadd_m(svbool_t pg, Packet a, Packet b, Packet c) __arm_streaming { + return svmla_m(pg, c, a, b); +} +// Sum of the active lanes. +template <typename Packet> +EIGEN_ALWAYS_INLINE typename sme_unpacket_traits<Packet>::type predux(svbool_t pg, Packet a) __arm_streaming { + return svaddv(pg, a); +} + +// Vector tuples: the lane of a tuple is an instruction immediate, hence a template parameter. +template <int Lane> +EIGEN_ALWAYS_INLINE svfloat32_t pget(svfloat32x2_t v) __arm_streaming { + return svget2_f32(v, Lane); +} +template <int Lane> +EIGEN_ALWAYS_INLINE svfloat32_t pget(svfloat32x4_t v) __arm_streaming { + return svget4_f32(v, Lane); +} +#ifdef EIGEN_VECTORIZE_SME_F64F64 +template <int Lane> +EIGEN_ALWAYS_INLINE svfloat64_t pget(svfloat64x2_t v) __arm_streaming { + return svget2_f64(v, Lane); +} +template <int Lane> +EIGEN_ALWAYS_INLINE svfloat64_t pget(svfloat64x4_t v) __arm_streaming { + return svget4_f64(v, Lane); +} +#endif +template <typename Packet> +EIGEN_ALWAYS_INLINE auto pcreate(Packet a, Packet b) __arm_streaming -> decltype(svcreate2(a, b)) { + return svcreate2(a, b); +} +template <typename Packet> +EIGEN_ALWAYS_INLINE auto pcreate(Packet a, Packet b, Packet c, Packet d) __arm_streaming + -> decltype(svcreate4(a, b, c, d)) { + return svcreate4(a, b, c, d); +} + +// Permutes: puzp1/puzp2 gather the even/odd lanes of the concatenation (a, b), pzip1/pzip2 +// interleave its low/high halves, psplice(pg, a, b) is the active lanes of a followed by the leading +// lanes of b. +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet puzp1(Packet a, Packet b) __arm_streaming { + return svuzp1(a, b); +} +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet puzp2(Packet a, Packet b) __arm_streaming { + return svuzp2(a, b); +} +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet pzip1(Packet a, Packet b) __arm_streaming { + return svzip1(a, b); +} +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet pzip2(Packet a, Packet b) __arm_streaming { + return svzip2(a, b); +} +template <typename Packet> +EIGEN_ALWAYS_INLINE Packet psplice(svbool_t pg, Packet a, Packet b) __arm_streaming { + return svsplice(pg, a, b); +} + +// Entering and leaving streaming mode sets FPSR's cumulative exception flags (Arm DDI0616, RMHTLZ), also when the OS +// resumes a thread in streaming mode, and FP arithmetic into ZA raises none. So a call into streaming code keeps the +// caller's flags and reports none of its own. +struct sme_fpsr_guard { + EIGEN_ALWAYS_INLINE sme_fpsr_guard() { asm volatile("mrs %0, fpsr" : "=r"(value) : : "memory"); } + EIGEN_ALWAYS_INLINE ~sme_fpsr_guard() { asm volatile("msr fpsr, %0" : : "r"(value) : "memory"); } + sme_fpsr_guard(const sme_fpsr_guard&) = delete; + sme_fpsr_guard& operator=(const sme_fpsr_guard&) = delete; + + private: + std::uint64_t value; +}; + +// min() usable from streaming functions (numext::mini lacks the __arm_streaming_compatible attribute). +template <typename T> +EIGEN_ALWAYS_INLINE T sme_min(T a, T b) __arm_streaming_compatible { + return a < b ? a : b; +} + +// Offset a pointer by n elements without forming the pointer value: an access whose predicate is +// empty makes no memory reference, but computing an address more than one past the end of the +// object is undefined regardless, so the second-vector accesses of the kernels reach their address +// through uintptr_t. +template <typename T> +EIGEN_ALWAYS_INLINE T* sme_offset(T* p, Index n) __arm_streaming_compatible { + return reinterpret_cast<T*>(uintptr_t(p) + ptrdiff_t(n) * sizeof(T)); +} + +// ZA tile access. A tile number is an instruction immediate, hence a template parameter; a slice +// number is a register operand and stays a value. The element width follows the vector or pointer +// argument, or is given as the scalar type where there is neither (sme_za_read). +template <int Tile> +EIGEN_ALWAYS_INLINE void sme_ld1_hor_za(uint32_t slice, svbool_t pg, const float* p) __arm_streaming __arm_inout("za") { + svld1_hor_za32(Tile, slice, pg, p); +} +template <int Tile> +EIGEN_ALWAYS_INLINE svfloat32_t sme_read_hor_za(svfloat32_t zero, svbool_t pg, + uint32_t slice) __arm_streaming __arm_in("za") { + return svread_hor_za32_f32_m(zero, pg, Tile, slice); +} +template <int Tile> +EIGEN_ALWAYS_INLINE svfloat32_t sme_read_ver_za(svfloat32_t zero, svbool_t pg, + uint32_t slice) __arm_streaming __arm_in("za") { + return svread_ver_za32_f32_m(zero, pg, Tile, slice); +} +// Four slices at once (SME2 MOVA ... vg4). +template <int Tile> +EIGEN_ALWAYS_INLINE void sme_write_hor_za_vg4(uint32_t slice, svfloat32_t a, svfloat32_t b, svfloat32_t c, + svfloat32_t d) __arm_streaming __arm_inout("za") { + svwrite_hor_za32_f32_vg4(Tile, slice, svcreate4_f32(a, b, c, d)); +} +template <int Tile> +EIGEN_ALWAYS_INLINE void sme_write_ver_za_vg4(uint32_t slice, svfloat32x4_t v) __arm_streaming __arm_inout("za") { + svwrite_ver_za32_f32_vg4(Tile, slice, v); +} +// Outer product accumulate (FMOPA) and subtract (FMOPS) into a tile. +template <int Tile> +EIGEN_ALWAYS_INLINE void sme_mopa(svbool_t pm, svbool_t pn, svfloat32_t a, + svfloat32_t b) __arm_streaming __arm_inout("za") { + svmopa_za32_f32_m(Tile, pm, pn, a, b); +} +template <int Tile> +EIGEN_ALWAYS_INLINE void sme_mops(svbool_t pm, svbool_t pn, svfloat32_t a, + svfloat32_t b) __arm_streaming __arm_inout("za") { + svmops_za32_f32_m(Tile, pm, pn, a, b); +} +// SME2 multi-vector FMLA into four ZA array vectors, the vector-length accumulators of the vector +// kernels; four independent groups hide the accumulator latency. +EIGEN_ALWAYS_INLINE void sme_madd_za_vg1x4(uint32_t slice, svfloat32x4_t x, + svfloat32_t y) __arm_streaming __arm_inout("za") { + svmla_single_za32_f32_vg1x4(slice, x, y); +} +EIGEN_ALWAYS_INLINE void sme_madd_za_vg1x4(uint32_t slice, svfloat32x4_t x, + svfloat32x4_t y) __arm_streaming __arm_inout("za") { + svmla_za32_f32_vg1x4(slice, x, y); +} +EIGEN_ALWAYS_INLINE void sme_write_za_vg1x4(uint32_t slice, svfloat32x4_t x) __arm_streaming __arm_inout("za") { + svwrite_za32_f32_vg1x4(slice, x); +} + +#ifdef EIGEN_VECTORIZE_SME_F64F64 +template <int Tile> +EIGEN_ALWAYS_INLINE void sme_ld1_hor_za(uint32_t slice, svbool_t pg, + const double* p) __arm_streaming __arm_inout("za") { + svld1_hor_za64(Tile, slice, pg, p); +} +template <int Tile> +EIGEN_ALWAYS_INLINE svfloat64_t sme_read_hor_za(svfloat64_t zero, svbool_t pg, + uint32_t slice) __arm_streaming __arm_in("za") { + return svread_hor_za64_f64_m(zero, pg, Tile, slice); +} +template <int Tile> +EIGEN_ALWAYS_INLINE svfloat64_t sme_read_ver_za(svfloat64_t zero, svbool_t pg, + uint32_t slice) __arm_streaming __arm_in("za") { + return svread_ver_za64_f64_m(zero, pg, Tile, slice); +} +template <int Tile> +EIGEN_ALWAYS_INLINE void sme_write_hor_za_vg4(uint32_t slice, svfloat64_t a, svfloat64_t b, svfloat64_t c, + svfloat64_t d) __arm_streaming __arm_inout("za") { + svwrite_hor_za64_f64_vg4(Tile, slice, svcreate4_f64(a, b, c, d)); +} +template <int Tile> +EIGEN_ALWAYS_INLINE void sme_write_ver_za_vg4(uint32_t slice, svfloat64x4_t v) __arm_streaming __arm_inout("za") { + svwrite_ver_za64_f64_vg4(Tile, slice, v); +} +template <int Tile> +EIGEN_ALWAYS_INLINE void sme_mopa(svbool_t pm, svbool_t pn, svfloat64_t a, + svfloat64_t b) __arm_streaming __arm_inout("za") { + svmopa_za64_f64_m(Tile, pm, pn, a, b); +} +template <int Tile> +EIGEN_ALWAYS_INLINE void sme_mops(svbool_t pm, svbool_t pn, svfloat64_t a, + svfloat64_t b) __arm_streaming __arm_inout("za") { + svmops_za64_f64_m(Tile, pm, pn, a, b); +} +EIGEN_ALWAYS_INLINE void sme_madd_za_vg1x4(uint32_t slice, svfloat64x4_t x, + svfloat64_t y) __arm_streaming __arm_inout("za") { + svmla_single_za64_f64_vg1x4(slice, x, y); +} +EIGEN_ALWAYS_INLINE void sme_madd_za_vg1x4(uint32_t slice, svfloat64x4_t x, + svfloat64x4_t y) __arm_streaming __arm_inout("za") { + svmla_za64_f64_vg1x4(slice, x, y); +} +EIGEN_ALWAYS_INLINE void sme_write_za_vg1x4(uint32_t slice, svfloat64x4_t x) __arm_streaming __arm_inout("za") { + svwrite_za64_f64_vg1x4(slice, x); +} +#endif // EIGEN_VECTORIZE_SME_F64F64 + +// The two reads with no width-carrying argument: four vertical slices of a tile, and four ZA array +// vectors. +template <typename Scalar> +struct sme_za_read; +template <> +struct sme_za_read<float> { + template <int Tile> + static EIGEN_ALWAYS_INLINE svfloat32x4_t ver_vg4(uint32_t slice) __arm_streaming __arm_in("za") { + return svread_ver_za32_f32_vg4(Tile, slice); + } + static EIGEN_ALWAYS_INLINE svfloat32x4_t vg1x4(uint32_t slice) __arm_streaming __arm_in("za") { + return svread_za32_f32_vg1x4(slice); + } +}; +#ifdef EIGEN_VECTORIZE_SME_F64F64 +template <> +struct sme_za_read<double> { + template <int Tile> + static EIGEN_ALWAYS_INLINE svfloat64x4_t ver_vg4(uint32_t slice) __arm_streaming __arm_in("za") { + return svread_ver_za64_f64_vg4(Tile, slice); + } + static EIGEN_ALWAYS_INLINE svfloat64x4_t vg1x4(uint32_t slice) __arm_streaming __arm_in("za") { + return svread_za64_f64_vg1x4(slice); + } +}; +#endif +template <int Tile, typename Scalar> +EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type_x4 sme_read_ver_za_vg4( + uint32_t slice) __arm_streaming __arm_in("za") { + return sme_za_read<Scalar>::template ver_vg4<Tile>(slice); +} +template <typename Scalar> +EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type_x4 sme_read_za_vg1x4( + uint32_t slice) __arm_streaming __arm_in("za") { + return sme_za_read<Scalar>::vg1x4(slice); +} + +// Outer-product accumulate with a compile-time sign: the complex kernel's four real products differ +// only in whether they add or subtract into the tile. +template <int Tile, bool Subtract, typename Packet> +EIGEN_ALWAYS_INLINE void sme_mopa_signed(svbool_t pm, svbool_t pn, Packet a, + Packet b) __arm_streaming __arm_inout("za") { + EIGEN_IF_CONSTEXPR (Subtract) { + sme_mops<Tile>(pm, pn, a, b); + } else { + sme_mopa<Tile>(pm, pn, a, b); + } +} + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_PACKET_MATH_SME_H
diff --git a/Eigen/src/Core/arch/SME/VectorKernels.h b/Eigen/src/Core/arch/SME/VectorKernels.h new file mode 100644 index 0000000..5aec6e9 --- /dev/null +++ b/Eigen/src/Core/arch/SME/VectorKernels.h
@@ -0,0 +1,288 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 +#ifndef EIGEN_SME_VECTOR_KERNELS_H +#define EIGEN_SME_VECTOR_KERNELS_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { +namespace internal { + +#if EIGEN_COMP_CLANG +#define EIGEN_SME_VECTOR_UNROLL4 _Pragma("unroll") +#elif EIGEN_COMP_GNUC +#define EIGEN_SME_VECTOR_UNROLL4 _Pragma("GCC unroll 4") +#else +#define EIGEN_SME_VECTOR_UNROLL4 +#endif + +// FPCR.RMode == 0b10 (roundTowardNegative), the one mode where (+0) + (-0) is -0. +static EIGEN_ALWAYS_INLINE bool sme_rounds_toward_negative() { + std::uint64_t fpcr; + asm volatile("mrs %0, fpcr" : "=r"(fpcr)); + return ((fpcr >> 22) & 3) == 2; +} + +template <typename Scalar, typename Index> +__arm_new("za") __arm_locally_streaming + EIGEN_DONT_INLINE void sme_axpy(Index n, const Scalar* x, Scalar* y, Scalar alpha, Index prefix) { + using Traits = sme_packet_traits<Scalar>; + using Vec = typename Traits::type; + const Index lanes = Traits::size(); + const auto pn = Traits::ptrue_c(); + const auto a = pset1<Vec>(alpha); + Index i = 0; + for (; i < prefix; i += prefix - i < lanes ? prefix - i : lanes) { + auto active = Traits::whilelt(i, prefix); + pstoreu(active, y + i, pmadd(active, ploadu(active, x + i), a, ploadu(active, y + i))); + } + for (; i <= n - 16 * lanes; i += 16 * lanes) { + EIGEN_SME_VECTOR_UNROLL4 + for (int k = 0; k < 4; ++k) { + auto xv = ploadu_x4(pn, x + i + k * 4 * lanes); + sme_write_za_vg1x4(k, ploadu_x4(pn, y + i + k * 4 * lanes)); + sme_madd_za_vg1x4(k, xv, a); + } + EIGEN_SME_VECTOR_UNROLL4 + for (int k = 0; k < 4; ++k) pstoreu_x4(pn, y + i + k * 4 * lanes, sme_read_za_vg1x4<Scalar>(k)); + } + for (; i <= n - 4 * lanes; i += 4 * lanes) { + auto xv = ploadu_x4(pn, x + i); + sme_write_za_vg1x4(0, ploadu_x4(pn, y + i)); + sme_madd_za_vg1x4(0, xv, a); + pstoreu_x4(pn, y + i, sme_read_za_vg1x4<Scalar>(0)); + } + for (; i < n; i += n - i < lanes ? n - i : lanes) { + auto tail = Traits::whilelt(i, n); + pstoreu(tail, y + i, pmadd(tail, ploadu(tail, x + i), a, ploadu(tail, y + i))); + } +} + +template <typename Scalar, typename Index> +__arm_new("za") __arm_locally_streaming EIGEN_DONT_INLINE Scalar sme_dot(Index n, const Scalar* x, const Scalar* y) { + using Traits = sme_packet_traits<Scalar>; + using Vec = typename Traits::type; + const Index lanes = Traits::size(); + const auto pg = Traits::ptrue(); + const auto pn = Traits::ptrue_c(); + // -0 is the additive identity except under roundTowardNegative, so a zero sum gets the IEEE sign. + const auto negative_zero = pset1<Vec>(Scalar(-0.0)); + EIGEN_SME_VECTOR_UNROLL4 + for (int k = 0; k < 4; ++k) + sme_write_za_vg1x4(k, pcreate(negative_zero, negative_zero, negative_zero, negative_zero)); + Index i = 0; + for (; i <= n - 16 * lanes; i += 16 * lanes) { + EIGEN_SME_VECTOR_UNROLL4 + for (int k = 0; k < 4; ++k) + sme_madd_za_vg1x4(k, ploadu_x4(pn, x + i + k * 4 * lanes), ploadu_x4(pn, y + i + k * 4 * lanes)); + } + for (; i <= n - 4 * lanes; i += 4 * lanes) sme_madd_za_vg1x4(0, ploadu_x4(pn, x + i), ploadu_x4(pn, y + i)); + auto tail_accumulator = negative_zero; + for (; i < n; i += n - i < lanes ? n - i : lanes) { + auto tail = Traits::whilelt(i, n); + tail_accumulator = pmadd_m(tail, ploadu(tail, x + i), ploadu(tail, y + i), tail_accumulator); + } + auto sum = tail_accumulator; + EIGEN_SME_VECTOR_UNROLL4 + for (int k = 0; k < 4; ++k) { + auto v = sme_read_za_vg1x4<Scalar>(k); + sum = padd(pg, sum, padd(pg, padd(pg, pget<0>(v), pget<1>(v)), padd(pg, pget<2>(v), pget<3>(v)))); + } + return predux(pg, sum); +} + +// y += alpha * ZA group k, updated in ZA like the accumulation. +template <typename Scalar, typename Vec> +static EIGEN_ALWAYS_INLINE void sme_gemv_update(unsigned int k, svcount_t active, Scalar* y, + Vec alpha) __arm_streaming __arm_inout("za") { + const auto sum = sme_read_za_vg1x4<Scalar>(k); + sme_write_za_vg1x4(k, ploadu_x4(active, y)); + sme_madd_za_vg1x4(k, sum, alpha); + pstoreu_x4(active, y, sme_read_za_vg1x4<Scalar>(k)); +} + +template <typename Scalar, typename Index> +__arm_new("za") __arm_locally_streaming + EIGEN_DONT_INLINE void sme_gemv(Index rows, Index cols, const Scalar* a, Index stride, const Scalar* x, Scalar* y, + Scalar alpha, Index block_cols) { + using Traits = sme_packet_traits<Scalar>; + using Vec = typename Traits::type; + const Index lanes = Traits::size(); + const auto scale = pset1<Vec>(alpha); + // Match the generic GEMV's scaled column batches to bound the unscaled sums. + for (Index first = 0; first < cols;) { + const Index end = first + (cols - first < block_cols ? cols - first : block_cols); + // Four predicated chunks per row block keep four independent FMLA chains, also in the final partial block. + // An inactive chunk addresses the block start and accesses no memory. + for (Index i = 0; i < rows;) { + const Index remaining = rows - i; + const auto p0 = Traits::whilelt_c4(i, rows), p1 = Traits::whilelt_c4(i + std::int64_t(4) * lanes, rows), + p2 = Traits::whilelt_c4(i + std::int64_t(8) * lanes, rows), + p3 = Traits::whilelt_c4(i + std::int64_t(12) * lanes, rows); + const Index o1 = remaining > 4 * lanes ? 4 * lanes : 0, o2 = remaining > 8 * lanes ? 8 * lanes : 0, + o3 = remaining > 12 * lanes ? 12 * lanes : 0; + svzero_za(); + for (Index j = first; j < end; ++j) { + const auto b = pset1<Vec>(x[j]); + const Scalar* column = a + i + j * stride; + sme_madd_za_vg1x4(0, ploadu_x4(p0, column), b); + sme_madd_za_vg1x4(1, ploadu_x4(p1, column + o1), b); + sme_madd_za_vg1x4(2, ploadu_x4(p2, column + o2), b); + sme_madd_za_vg1x4(3, ploadu_x4(p3, column + o3), b); + } + sme_gemv_update(0, p0, y + i, scale); + sme_gemv_update(1, p1, y + i + o1, scale); + sme_gemv_update(2, p2, y + i + o2, scale); + sme_gemv_update(3, p3, y + i + o3, scale); + // Bound the final increment to avoid signed overflow with a 32-bit Index near its maximum. + i += remaining < 16 * lanes ? remaining : 16 * lanes; + } + first = end; + } +} + +#undef EIGEN_SME_VECTOR_UNROLL4 + +template <typename Scalar> +struct sme_vector_scalar : false_type {}; +template <> +struct sme_vector_scalar<float> : true_type {}; +#ifdef EIGEN_VECTORIZE_SME_F64F64 +template <> +struct sme_vector_scalar<double> : true_type {}; +#endif + +template <typename Xpr> +struct sme_vector_access : bool_constant<(sme_vector_scalar<typename traits<Xpr>::Scalar>::value && + bool(Xpr::IsVectorAtCompileTime) && has_direct_access<Xpr>::value)> {}; + +template <typename Lhs, typename Rhs> +struct sme_dot_supported : bool_constant<sme_vector_access<Lhs>::value && sme_vector_access<Rhs>::value && + is_same<typename traits<Lhs>::Scalar, typename traits<Rhs>::Scalar>::value> {}; + +// Fixed crossovers, fitted on Apple M4 (SVL 512): below them the NEON/SME memory handoff outweighs the +// streaming kernels. Core's cache estimates, including setCpuCacheSizes overrides, supply the others. +static constexpr std::size_t kSmeDotMinBytes = 16384; // per operand +static constexpr std::size_t kSmeAxpyMinBytes = 4096; // per operand +static constexpr Index kSmeGemvMinRows = 128; +static constexpr Index kSmeGemvMinCols = 4; +// A GEMV whose RHS reaches kSmeGemvWideRhsBytes needs only kSmeGemvWideMinBytes of matrix, not Core's L1 estimate. +static constexpr std::size_t kSmeGemvWideRhsBytes = 128; +static constexpr std::size_t kSmeGemvWideMinBytes = 32768; + +// Keep cache queries out of callers so the small vector fallback can inline. +// Per operand, the L1 estimate (divided by L1Divisor) is the lower crossover and the L2 estimate the upper one. +template <typename Scalar, int L1Divisor = 1> +EIGEN_DONT_INLINE bool sme_vector_size_suitable(Index size) { + std::ptrdiff_t l1, l2, l3; + manage_caching_sizes(GetAction, &l1, &l2, &l3); + return l1 > 0 && l2 > 0 && size >= 0 && + static_cast<std::size_t>(size) >= (static_cast<std::size_t>(l1) - 1) / (L1Divisor * sizeof(Scalar)) + 1 && + static_cast<std::size_t>(size) <= static_cast<std::size_t>(l2) / sizeof(Scalar); +} + +template <typename Lhs, typename Rhs> +struct default_inner_product_impl<Lhs, Rhs, true, std::enable_if_t<sme_dot_supported<Lhs, Rhs>::value>> + : default_inner_product_impl<Lhs, Rhs, true, false_type> { + using Base = default_inner_product_impl<Lhs, Rhs, true, false_type>; + using Scalar = typename traits<Lhs>::Scalar; + static EIGEN_STRONG_INLINE Scalar run(const MatrixBase<Lhs>& lhs, const MatrixBase<Rhs>& rhs) { + inner_product_assert<Lhs, Rhs>::run(lhs.derived(), rhs.derived()); + // DOT has no streaming stores for a following NEON consumer: half L1 per operand suffices. + if (lhs.size() >= Index(kSmeDotMinBytes / sizeof(Scalar)) && sme_vector_size_suitable<Scalar, 2>(lhs.size()) && + lhs.innerStride() == 1 && rhs.innerStride() == 1) { + sme_fpsr_guard status; + const Scalar result = sme_dot(lhs.size(), lhs.derived().data(), rhs.derived().data()); + // Under roundTowardNegative the -0 seeds turn +0 sums into -0. + if (result != Scalar(0) || !sme_rounds_toward_negative()) return result; + } + return Base::run(lhs, rhs); + } +}; + +template <typename Dst, typename Src> +EIGEN_STRONG_INLINE bool sme_try_axpy(Dst& dst, const Src& src, typename traits<Src>::Scalar alpha) { + using Scalar = typename traits<Src>::Scalar; + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); + if (dst.size() < Index(kSmeAxpyMinBytes / sizeof(Scalar)) || !sme_vector_size_suitable<Scalar>(dst.size()) || + dst.innerStride() != 1 || src.innerStride() != 1) + return false; + const std::uintptr_t dst_address = reinterpret_cast<std::uintptr_t>(dst.data()); + const std::uintptr_t src_address = reinterpret_cast<std::uintptr_t>(src.data()); + const std::uintptr_t distance = dst_address > src_address ? dst_address - src_address : src_address - dst_address; + // Exact aliasing is coefficient-wise; partial overlap must retain the default traversal. + if (distance != 0 && distance / sizeof(Scalar) < static_cast<std::uintptr_t>(dst.size())) return false; + // Align streaming stores to 64 bytes without changing Eigen's allocation alignment. + sme_fpsr_guard status; + sme_axpy(dst.size(), src.data(), dst.data(), alpha, first_aligned<64>(dst.data(), dst.size())); + return true; +} + +template <typename Dst, typename Scalar, typename Lhs, typename Rhs> +struct Assignment< + Dst, CwiseBinaryOp<scalar_product_op<Scalar, Scalar>, Lhs, Rhs>, add_assign_op<Scalar, Scalar>, Dense2Dense, + std::enable_if_t<sme_vector_access<Dst>::value && + (sme_vector_access<Lhs>::value || sme_vector_access<Rhs>::value) && + blas_traits<CwiseBinaryOp<scalar_product_op<Scalar, Scalar>, Lhs, Rhs>>::HasScalarFactor && + is_same<Scalar, typename traits<Dst>::Scalar>::value>> { + using Source = CwiseBinaryOp<scalar_product_op<Scalar, Scalar>, Lhs, Rhs>; + using BlasTraits = blas_traits<Source>; + static EIGEN_STRONG_INLINE void run(Dst& dst, const Source& src, const add_assign_op<Scalar, Scalar>& func) { + // A direct operand restricts extraction to one scale, preserving nested products' evaluation order. + if (!sme_try_axpy(dst, BlasTraits::extract(src), BlasTraits::extractScalarFactor(src))) + Assignment<Dst, Source, add_assign_op<Scalar, Scalar>, Dense2Dense, false_type>::run(dst, src, func); + } +}; + +#ifndef EIGEN_USE_BLAS +template <typename Scalar, typename Index> +EIGEN_STRONG_INLINE bool sme_gemv_size_suitable(Index rows, Index cols) { + if (rows < Index(kSmeGemvMinRows) || cols < Index(kSmeGemvMinCols)) return false; + if (cols >= Index(kSmeGemvWideRhsBytes / sizeof(Scalar)) && + cols > Index(kSmeGemvWideMinBytes / sizeof(Scalar) - 1) / rows) + return true; + // Thin products amortize the NEON/SME handoff once the matrix reaches Core's L1 estimate. + const std::ptrdiff_t l1 = l1CacheSize(); + return l1 > 0 && static_cast<std::size_t>(cols) > + (static_cast<std::size_t>(l1) - 1) / sizeof(Scalar) / static_cast<std::size_t>(rows); +} + +#define EIGEN_SME_GEMV_SPECIALIZATION(Scalar) \ + template <typename Index, bool ConjugateLhs, bool ConjugateRhs> \ + struct general_matrix_vector_product<Index, Scalar, const_blas_data_mapper<Scalar, Index, ColMajor>, ColMajor, \ + ConjugateLhs, Scalar, const_blas_data_mapper<Scalar, Index, RowMajor>, \ + ConjugateRhs, Specialized> { \ + static EIGEN_STRONG_INLINE void run(Index rows, Index cols, \ + const const_blas_data_mapper<Scalar, Index, ColMajor>& lhs, \ + const const_blas_data_mapper<Scalar, Index, RowMajor>& rhs, Scalar* res, \ + Index resIncr, Scalar alpha) { \ + if (sme_gemv_size_suitable<Scalar>(rows, cols) && rhs.stride() == 1 && resIncr == 1) { \ + const std::ptrdiff_t l1 = l1CacheSize(); \ + const Index block_cols = cols < 128 ? cols \ + : (l1 > 0 && static_cast<std::size_t>(lhs.stride()) <= \ + (static_cast<std::size_t>(l1) - 1) / sizeof(Scalar) \ + ? Index(16) \ + : Index(4)); \ + /* BLAS contract: alpha == 0 leaves the result unchanged. */ \ + if (alpha == Scalar(0)) return; \ + sme_fpsr_guard status; \ + sme_gemv(rows, cols, lhs.data(), lhs.stride(), rhs.data(), res, alpha, block_cols); \ + } else { \ + general_matrix_vector_product<Index, Scalar, const_blas_data_mapper<Scalar, Index, ColMajor>, ColMajor, \ + ConjugateLhs, Scalar, const_blas_data_mapper<Scalar, Index, RowMajor>, \ + ConjugateRhs, BuiltIn>::run(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ + } \ + }; +EIGEN_SME_GEMV_SPECIALIZATION(float) +#ifdef EIGEN_VECTORIZE_SME_F64F64 +EIGEN_SME_GEMV_SPECIALIZATION(double) +#endif +#undef EIGEN_SME_GEMV_SPECIALIZATION +#endif + +} // namespace internal +} // namespace Eigen + +#endif
diff --git a/Eigen/src/Core/arch/SSE/PacketMath.h b/Eigen/src/Core/arch/SSE/PacketMath.h index ff3d13a..cd1dcf2 100644 --- a/Eigen/src/Core/arch/SSE/PacketMath.h +++ b/Eigen/src/Core/arch/SSE/PacketMath.h
@@ -1721,20 +1721,19 @@ // supported by SSE, and has more range than is needed for exponents. template <> EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) { - // Clamp exponent to [-2099, 2099] + // The single-rounding split of pldexp_generic on the two low int32 lanes. Interleaving t and e - t puts each + // biased pair in one 64-bit lane, t + 2 * bias low: shifting left by 51 drops the high half, and shifting right by + // 32 first selects it. const Packet2d max_exponent = pset1<Packet2d>(2099.0); - const Packet2d e = pmin(pmax(exponent, pnegate(max_exponent)), max_exponent); - - // Convert e to integer and swizzle to low-order bits. - const Packet4i ei = vec4i_swizzle1(_mm_cvtpd_epi32(e), 0, 3, 1, 3); - - // Preserve the sequential 4-way split; see pldexp_generic. - const Packet4i bias = _mm_set_epi32(0, 1023, 0, 1023); - const Packet4i b = parithmetic_shift_right<2>(ei); // floor(e/4) - const Packet4i b_remainder = psub(psub(ei, b), padd(b, b)); // e - 3b (depth 2) - const Packet2d c1 = _mm_castsi128_pd(_mm_slli_epi64(padd(b, bias), 52)); // 2^b - const Packet2d c2 = _mm_castsi128_pd(_mm_slli_epi64(padd(b_remainder, bias), 52)); // 2^(e - 3b) - return pmul(pmul(pmul(pmul(a, c1), c1), c1), c2); // a * 2^e + const Packet2d last_max = pset1<Packet2d>(1022.0); + const Packet4i e = _mm_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); + const Packet4i b = _mm_cvtpd_epi32(pmin(pmax(exponent, pnegate(last_max)), last_max)); + const Packet4i t = pandnot(psub(e, b), pset1<Packet4i>(1)); // even + const Packet4i biased = + padd(Packet4i(_mm_unpacklo_epi32(t, psub(e, t))), Packet4i(_mm_set_epi32(1023, 2046, 1023, 2046))); + const Packet2d c1 = _mm_castsi128_pd(_mm_slli_epi64(biased, 51)); // 2^(t/2) + const Packet2d c2 = _mm_castsi128_pd(_mm_slli_epi64(_mm_srli_epi64(biased, 32), 52)); // 2^(e - t) + return pldexp_apply_factors(a, c1, c2); // a * 2^e } // We specialize pldexp here, since the generic implementation uses Packet2l, which is not well
diff --git a/Eigen/src/Core/arch/SVE/Complex.h b/Eigen/src/Core/arch/SVE/Complex.h index f9cb5ca..9f5a58b 100644 --- a/Eigen/src/Core/arch/SVE/Complex.h +++ b/Eigen/src/Core/arch/SVE/Complex.h
@@ -486,6 +486,40 @@ EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(PacketXcf, PacketXf) EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(PacketXcd, PacketXd) +/*---------------- load/store segment support ----------------*/ + +// A complex lane is two real lanes, so the predicate is formed over 2 * begin and 2 * count. + +template <> +struct has_packet_segment<PacketXcf> : std::true_type {}; + +template <> +inline PacketXcf ploaduSegment<PacketXcf>(const std::complex<float>* from, Index begin, Index count) { + return PacketXcf(svld1_f32(sve_segment_predicate_b32(2 * begin, 2 * count), reinterpret_cast<const float*>(from))); +} + +template <> +inline void pstoreuSegment<std::complex<float>, PacketXcf>(std::complex<float>* to, const PacketXcf& from, Index begin, + Index count) { + svst1_f32(sve_segment_predicate_b32(2 * begin, 2 * count), reinterpret_cast<float*>(to), from.v); +} + +template <> +struct has_packet_segment<PacketXcd> : std::true_type {}; + +template <> +inline PacketXcd ploaduSegment<PacketXcd>(const std::complex<double>* from, Index begin, Index count) { + return PacketXcd(svld1_f64(sve_segment_predicate_b64(2 * begin, 2 * count), reinterpret_cast<const double*>(from))); +} + +template <> +inline void pstoreuSegment<std::complex<double>, PacketXcd>(std::complex<double>* to, const PacketXcd& from, + Index begin, Index count) { + svst1_f64(sve_segment_predicate_b64(2 * begin, 2 * count), reinterpret_cast<double*>(to), from.v); +} + +/*---------------- end load/store segment support ----------------*/ + } // end namespace internal } // end namespace Eigen
diff --git a/Eigen/src/Core/arch/SVE/PacketMath.h b/Eigen/src/Core/arch/SVE/PacketMath.h index cbdbd20..13dd52f 100644 --- a/Eigen/src/Core/arch/SVE/PacketMath.h +++ b/Eigen/src/Core/arch/SVE/PacketMath.h
@@ -989,7 +989,8 @@ template <> EIGEN_STRONG_INLINE PacketXf pldexp<PacketXf>(const PacketXf& a, const PacketXf& exponent) { - return pldexp_generic(a, exponent); + // fscale is a * 2^n rounded once; the conversion saturates, and a saturated n over- or underflows every finite a. + return svscale_f32_x(svptrue_b32(), a, svcvt_s32_f32_x(svptrue_b32(), exponent)); } template <> @@ -1367,7 +1368,7 @@ template <> EIGEN_STRONG_INLINE PacketXd pldexp<PacketXd>(const PacketXd& a, const PacketXd& exponent) { - return pldexp_generic(a, exponent); + return svscale_f64_x(svptrue_b64(), a, svcvt_s64_f64_x(svptrue_b64(), exponent)); // see pldexp<PacketXf> } template <> @@ -1383,6 +1384,86 @@ return generic_rsqrt_newton_step<PacketXd, /*Steps=*/3>::run(a, svrsqrte_f64(a)); } +/*---------------- load/store segment support ----------------*/ + +// Predicates with lanes [begin, begin + count) active. begin is a literal 0 at the assignment +// call sites, so that branch folds and the common case is a single whilelt. +EIGEN_STRONG_INLINE svbool_t sve_segment_predicate_b32(Index begin, Index count) { + eigen_assert(begin >= 0 && count >= 0); + if (begin == 0) return svwhilelt_b32(uint64_t(0), uint64_t(count)); + return svnot_b_z(svwhilelt_b32(uint64_t(0), uint64_t(begin + count)), svwhilelt_b32(uint64_t(0), uint64_t(begin))); +} + +EIGEN_STRONG_INLINE svbool_t sve_segment_predicate_b64(Index begin, Index count) { + eigen_assert(begin >= 0 && count >= 0); + if (begin == 0) return svwhilelt_b64(uint64_t(0), uint64_t(count)); + return svnot_b_z(svwhilelt_b64(uint64_t(0), uint64_t(begin + count)), svwhilelt_b64(uint64_t(0), uint64_t(begin))); +} + +/*---------------- int32 ----------------*/ + +template <> +struct has_packet_segment<PacketXi> : std::true_type {}; + +template <> +inline PacketXi ploaduSegment<PacketXi>(const numext::int32_t* from, Index begin, Index count) { + return svld1_s32(sve_segment_predicate_b32(begin, count), from); +} + +template <> +inline void pstoreuSegment<numext::int32_t, PacketXi>(numext::int32_t* to, const PacketXi& from, Index begin, + Index count) { + svst1_s32(sve_segment_predicate_b32(begin, count), to, from); +} + +/*---------------- int64 ----------------*/ + +template <> +struct has_packet_segment<PacketXl> : std::true_type {}; + +template <> +inline PacketXl ploaduSegment<PacketXl>(const numext::int64_t* from, Index begin, Index count) { + return svld1_s64(sve_segment_predicate_b64(begin, count), from); +} + +template <> +inline void pstoreuSegment<numext::int64_t, PacketXl>(numext::int64_t* to, const PacketXl& from, Index begin, + Index count) { + svst1_s64(sve_segment_predicate_b64(begin, count), to, from); +} + +/*---------------- float ----------------*/ + +template <> +struct has_packet_segment<PacketXf> : std::true_type {}; + +template <> +inline PacketXf ploaduSegment<PacketXf>(const float* from, Index begin, Index count) { + return svld1_f32(sve_segment_predicate_b32(begin, count), from); +} + +template <> +inline void pstoreuSegment<float, PacketXf>(float* to, const PacketXf& from, Index begin, Index count) { + svst1_f32(sve_segment_predicate_b32(begin, count), to, from); +} + +/*---------------- double ----------------*/ + +template <> +struct has_packet_segment<PacketXd> : std::true_type {}; + +template <> +inline PacketXd ploaduSegment<PacketXd>(const double* from, Index begin, Index count) { + return svld1_f64(sve_segment_predicate_b64(begin, count), from); +} + +template <> +inline void pstoreuSegment<double, PacketXd>(double* to, const PacketXd& from, Index begin, Index count) { + svst1_f64(sve_segment_predicate_b64(begin, count), to, from); +} + +/*---------------- end load/store segment support ----------------*/ + } // namespace internal } // namespace Eigen
diff --git a/Eigen/src/Core/arch/ZVector/PacketMath.h b/Eigen/src/Core/arch/ZVector/PacketMath.h index 19ece0f..f0427e0 100644 --- a/Eigen/src/Core/arch/ZVector/PacketMath.h +++ b/Eigen/src/Core/arch/ZVector/PacketMath.h
@@ -1298,19 +1298,17 @@ template <> EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) { - // Clamp exponent to [-2099, 2099] + // The single-rounding split of pldexp_generic on int64 exponents. const Packet2d max_exponent = pset1<Packet2d>(2099.0); + const Packet2d last_max = pset1<Packet2d>(1022.0); const Packet2l e = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent)); - - // Split 2^e into four factors and multiply: + const Packet2l b = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(last_max)), last_max)); const Packet2l bias = {1023, 1023}; - Packet2l b = plogical_shift_right<2>(e); // floor(e/4) - Packet2d c = reinterpret_cast<Packet2d>(plogical_shift_left<52>(b + bias)); - Packet2d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b) - b = psub(psub(psub(e, b), b), b); // e - 3b - c = reinterpret_cast<Packet2d>(plogical_shift_left<52>(b + bias)); // 2^(e - 3b) - out = pmul(out, c); // a * 2^e - return out; + const Packet2l even = {-2, -2}; + const Packet2l t = psub(e, b) & even; + const Packet2d c1 = reinterpret_cast<Packet2d>(plogical_shift_left<51>(t + bias + bias)); // 2^(t/2) + const Packet2d c2 = reinterpret_cast<Packet2d>(plogical_shift_left<52>(psub(e, t) + bias)); // 2^(e - t) + return pldexp_apply_factors(a, c1, c2); // a * 2^e } template <>
diff --git a/Eigen/src/Core/functors/AssignmentFunctors.h b/Eigen/src/Core/functors/AssignmentFunctors.h index 01c0c2b..5a5f284 100644 --- a/Eigen/src/Core/functors/AssignmentFunctors.h +++ b/Eigen/src/Core/functors/AssignmentFunctors.h
@@ -27,12 +27,13 @@ EIGEN_DEVICE_FUNC constexpr void assignCoeff(DstScalar& a, const SrcScalar& b) const { a = b; } template <int Alignment, typename Packet> - EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { pstoret<DstScalar, Packet, Alignment>(a, b); } template <int Alignment, typename Packet> - EIGEN_STRONG_INLINE void assignPacketSegment(DstScalar* a, const Packet& b, Index begin, Index count) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketSegment(DstScalar* a, const Packet& b, Index begin, + Index count) const { pstoretSegment<DstScalar, Packet, Alignment>(a, b, begin, count); } }; @@ -62,13 +63,14 @@ } template <int Alignment, typename Packet> - EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const { assign_op<DstScalar, DstScalar>().template assignPacket<Alignment, Packet>( a, Func().packetOp(ploadt<Packet, Alignment>(a), b)); } template <int Alignment, typename Packet> - EIGEN_STRONG_INLINE void assignPacketSegment(DstScalar* a, const Packet& b, Index begin, Index count) const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketSegment(DstScalar* a, const Packet& b, Index begin, + Index count) const { assign_op<DstScalar, DstScalar>().template assignPacketSegment<Alignment, Packet>( a, Func().packetOp(ploadtSegment<Packet, Alignment>(a, begin, count), b), begin, count); }
diff --git a/Eigen/src/Core/functors/UnaryFunctors.h b/Eigen/src/Core/functors/UnaryFunctors.h index 7b9f7e9..34beb53 100644 --- a/Eigen/src/Core/functors/UnaryFunctors.h +++ b/Eigen/src/Core/functors/UnaryFunctors.h
@@ -1455,7 +1455,7 @@ return static_cast<Scalar>(pow(a, m_exponent)); } template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet packetOp(const Packet& a) const { return unary_pow_impl<Packet, Scalar>::run(a, m_exponent); } @@ -1464,16 +1464,36 @@ scalar_unary_pow_op() {} }; -template <typename Scalar, typename ExponentScalar, bool BaseIsInteger> -struct scalar_unary_pow_op<Scalar, ExponentScalar, BaseIsInteger, true, false, false> { +// Specialization for a complex base and a real, non-integer exponent type: integer-valued exponents take the +// repeated-squaring path, which is exact where the power is representable; the rest defer to pow. There is no +// vectorized complex pow, so this stays scalar. +template <typename Scalar, typename ExponentScalar> +struct scalar_unary_pow_op<Scalar, ExponentScalar, false, false, true, false> { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE scalar_unary_pow_op(const ExponentScalar& exponent) + : m_exponent(exponent), + m_use_repeated_squaring((numext::isfinite)(exponent) && numext::round(exponent) == exponent && + unary_pow::use_repeated_squaring<Scalar>(exponent)) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a) const { + if (m_use_repeated_squaring) return unary_pow::int_pow(a, m_exponent); + EIGEN_USING_STD(pow); + return static_cast<Scalar>(pow(a, m_exponent)); + } + + private: + const ExponentScalar m_exponent; + const bool m_use_repeated_squaring; + scalar_unary_pow_op() {} +}; + +template <typename Scalar, typename ExponentScalar, bool BaseIsInteger, bool BaseIsComplex> +struct scalar_unary_pow_op<Scalar, ExponentScalar, BaseIsInteger, true, BaseIsComplex, false> { EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE scalar_unary_pow_op(const ExponentScalar& exponent) : m_exponent(exponent) {} - // TODO: error handling logic for complex^real_integer EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE Scalar operator()(const Scalar& a) const { return unary_pow_impl<Scalar, ExponentScalar>::run(a, m_exponent); } template <typename Packet> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packetOp(const Packet& a) const { + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet packetOp(const Packet& a) const { return unary_pow_impl<Packet, ExponentScalar>::run(a, m_exponent); } @@ -1486,8 +1506,15 @@ struct functor_traits<scalar_unary_pow_op<Scalar, ExponentScalar>> { enum { GenPacketAccess = functor_traits<scalar_pow_op<Scalar, ExponentScalar>>::PacketAccess, - IntPacketAccess = !NumTraits<Scalar>::IsComplex && packet_traits<Scalar>::HasMul && - (packet_traits<Scalar>::HasDiv || NumTraits<Scalar>::IsInteger) && packet_traits<Scalar>::HasCmp, + // Only the real-exponent specializations define packetOp. Every path multiplies packets of the base type and + // divides them for a negative exponent unless the base is an integer; the compares and selects run on the + // real packet, which for a complex base is the component view of its packet. A complex base is vectorized + // only through the double-word path, so it also needs that path's packet support. + IntPacketAccess = + !NumTraits<ExponentScalar>::IsComplex && packet_traits<Scalar>::HasMul && + (packet_traits<Scalar>::HasDiv || NumTraits<Scalar>::IsInteger) && + packet_traits<typename NumTraits<Scalar>::Real>::HasCmp && + (!NumTraits<Scalar>::IsComplex || unary_pow::use_double_word<typename packet_traits<Scalar>::type>::value), PacketAccess = NumTraits<ExponentScalar>::IsInteger ? IntPacketAccess : (IntPacketAccess && GenPacketAccess), Cost = functor_traits<scalar_pow_op<Scalar, ExponentScalar>>::Cost };
diff --git a/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Eigen/src/Core/products/GeneralBlockPanelKernel.h index 358ffc0..0a4bec3 100644 --- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h +++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h
@@ -163,6 +163,9 @@ #ifndef EIGEN_SME_LHS_WORKING_SET_BUDGET_BYTES #define EIGEN_SME_LHS_WORKING_SET_BUDGET_BYTES (7 * 1024 * 1024) #endif +#ifndef EIGEN_SME_SINGLE_PASS_RHS_BUDGET_BYTES +#define EIGEN_SME_SINGLE_PASS_RHS_BUDGET_BYTES (4 * 1024 * 1024) +#endif template <typename LhsScalar, typename RhsScalar, typename Index> void evaluateProductBlockingSizesHeuristicForSme(Index& k, Index& m, Index& n) { @@ -179,10 +182,12 @@ constexpr Index sme_max_kc = static_cast<Index>(128); constexpr Index sme_packed_rhs_budget_bytes = static_cast<Index>(128 * 1024); constexpr Index sme_lhs_working_set_budget_bytes = static_cast<Index>(128 * 1024); + constexpr Index sme_single_pass_rhs_budget_bytes = static_cast<Index>(128 * 1024); #else constexpr Index sme_max_kc = static_cast<Index>(EIGEN_SME_MAX_KC); constexpr Index sme_packed_rhs_budget_bytes = static_cast<Index>(EIGEN_SME_PACKED_RHS_BUDGET_BYTES); constexpr Index sme_lhs_working_set_budget_bytes = static_cast<Index>(EIGEN_SME_LHS_WORKING_SET_BUDGET_BYTES); + constexpr Index sme_single_pass_rhs_budget_bytes = static_cast<Index>(EIGEN_SME_SINGLE_PASS_RHS_BUDGET_BYTES); #endif // Keep kc large enough to amortize SME setup and accumulation, but cap very @@ -192,12 +197,6 @@ const Index max_kc = (numext::maxi)(Index(1), sme_max_kc * Index(sizeof(float)) / Index(sizeof(LhsScalar))); k = (numext::mini)(k, max_kc); - // Bound the packed RHS strip so very wide matrices do not allocate an - // unbounded blockB panel. - Index nc = sme_packed_rhs_budget_bytes / (numext::maxi)(Index(1), k * Index(sizeof(RhsScalar))); - nc = (nc / nr) * nr; - n = (numext::mini)(n, (numext::maxi)(nr, nc)); - const Index block_b_hot_bytes = k * nr * Index(sizeof(RhsScalar)); const Index min_lhs_bytes = mr * k * Index(sizeof(LhsScalar)); const Index block_a_bytes = sme_lhs_working_set_budget_bytes > block_b_hot_bytes @@ -205,6 +204,15 @@ : min_lhs_bytes; Index mc = block_a_bytes / (k * Index(sizeof(LhsScalar))); mc = (mc / mr) * mr; + + // Bound the packed RHS strip so very wide matrices do not allocate an unbounded blockB panel. When all rows fit + // one LHS block, each packed RHS block is read once, so it stays L2-sized. + const Index rhs_budget = m <= mc ? (numext::mini)(sme_packed_rhs_budget_bytes, sme_single_pass_rhs_budget_bytes) + : sme_packed_rhs_budget_bytes; + Index nc = rhs_budget / (numext::maxi)(Index(1), k * Index(sizeof(RhsScalar))); + nc = (nc / nr) * nr; + n = (numext::mini)(n, (numext::maxi)(nr, nc)); + m = (numext::mini)(m, (numext::maxi)(mr, mc)); } #endif @@ -2536,9 +2544,8 @@ return l3; } -/** Set the cpu L1 and L2 cache sizes (in bytes). - * These values are use to adjust the size of the blocks - * for the algorithms working per blocks. +/** Set the cpu L1, L2 and L3 cache sizes (in bytes). + * These values are used to adjust blocking sizes and, on some backends, the size crossovers between kernels. * * \sa computeProductBlockingSizes */ inline void setCpuCacheSizes(std::ptrdiff_t l1, std::ptrdiff_t l2, std::ptrdiff_t l3) {
diff --git a/Eigen/src/Core/products/GeneralMatrixMatrix.h b/Eigen/src/Core/products/GeneralMatrixMatrix.h index 85091f6..f5a717b 100644 --- a/Eigen/src/Core/products/GeneralMatrixMatrix.h +++ b/Eigen/src/Core/products/GeneralMatrixMatrix.h
@@ -62,8 +62,77 @@ } }; +#ifdef EIGEN_VECTORIZE_SME +// Defined in arch/SME/GeneralBlockPanelKernel.h, included after this header: +// whether the SME kernel can read this ColMajor LHS block straight from its +// source instead of a packed panel. +template <typename Scalar, typename Index> +bool sme_direct_lhs_ok(Index lhsStride, Index rows, Index depth, Index cols); +// True for the unit-stride ColMajor mapper the GEMM driver hands the packers. +template <typename Mapper> +struct sme_direct_lhs_mapper : std::false_type {}; +template <typename Scalar, typename Index> +struct sme_direct_lhs_mapper<const_blas_data_mapper<Scalar, Index, ColMajor>> + : bool_constant<!NumTraits<Scalar>::IsComplex> {}; +// Runs the block on the in-place LHS when the kernel can take it; the false +// overload keeps the call out of every other instantiation. +template <typename Gebp, typename ResMapper, typename LhsMapper, typename Scalar, typename ResScalar, typename Index> +EIGEN_ALWAYS_INLINE bool sme_run_direct_lhs(std::true_type, Gebp& gebp, const ResMapper& res, const LhsMapper& lhs, + Index i2, Index k2, const Scalar* blockB, Index mc, Index kc, Index nc, + ResScalar alpha) { + if (!sme_direct_lhs_ok<Scalar>(lhs.stride(), mc, kc, nc)) return false; + gebp.run_direct_lhs(res, &lhs(i2, k2), lhs.stride(), blockB, mc, kc, nc, alpha); + return true; +} +template <typename Scalar, int LhsOrder, int RhsOrder, typename Index> +bool sme_tiny_gemm(Index rows, Index cols, Index depth, const Scalar* lhs, Index lhsStride, const Scalar* rhs, + Index rhsStride, Scalar* res, Index resIncr, Index resStride, Scalar alpha); +// Where the tiny-result kernel beats both the coeff-based product and the packed paths, as tuned on Apple M4: short +// depths keep the coeff-based product, and a full 2 * ps x 8 result over a long depth the SME kernel. The NEON +// small-block switches pin the paths they select, so they turn it off. +template <typename Scalar, typename Index> +EIGEN_ALWAYS_INLINE bool sme_tiny_gemm_wins(Index rows, Index cols, Index depth) { +#if defined(EIGEN_SME_NO_NEON_SMALL_BLOCKS) || defined(EIGEN_SME_FORCE_NEON_SMALL_BLOCKS) || defined(EIGEN_USE_BLAS) + EIGEN_UNUSED_VARIABLE(rows); + EIGEN_UNUSED_VARIABLE(cols); + EIGEN_UNUSED_VARIABLE(depth); + return false; +#else + const Index ps = Index(16 / sizeof(Scalar)); // scalars per NEON vector + if (rows < 2 || cols < 2 || rows > 2 * ps || cols > 8) return false; + if (depth < 16 && (depth < 6 || rows * cols < 24)) return false; + return !(cols == 8 && rows > ps && depth >= 2048); +#endif +} +// Real float and double on the SME kernel take the tiny-result kernel; the false overload keeps it out of every +// other pair, including double without FEAT_SME_F64F64. +template <typename LhsScalar, typename RhsScalar> +struct sme_tiny_gemm_pair + : bool_constant<std::is_same<LhsScalar, RhsScalar>::value && + (std::is_same<LhsScalar, float>::value || std::is_same<LhsScalar, double>::value) && + sme_has_gebp_kernel<LhsScalar, RhsScalar>::value> {}; +template <int LhsOrder, int RhsOrder, typename Scalar, typename Index> +EIGEN_ALWAYS_INLINE bool sme_run_tiny_gemm(std::true_type, Index rows, Index cols, Index depth, const Scalar* lhs, + Index lhsStride, const Scalar* rhs, Index rhsStride, Scalar* res, + Index resIncr, Index resStride, Scalar alpha) { + return sme_tiny_gemm<Scalar, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, resIncr, + resStride, alpha); +} +template <int LhsOrder, int RhsOrder, typename LhsScalar, typename RhsScalar, typename ResScalar, typename Index> +EIGEN_ALWAYS_INLINE bool sme_run_tiny_gemm(std::false_type, Index, Index, Index, const LhsScalar*, Index, + const RhsScalar*, Index, ResScalar*, Index, Index, ResScalar) { + return false; +} +template <typename Gebp, typename ResMapper, typename LhsMapper, typename Scalar, typename ResScalar, typename Index> +EIGEN_ALWAYS_INLINE bool sme_run_direct_lhs(std::false_type, Gebp&, const ResMapper&, const LhsMapper&, Index, Index, + const Scalar*, Index, Index, Index, ResScalar) { + return false; +} +#endif + // RHS-first loop order: nc -> kc -> mc. Used by SME to stream ColMajor result -// stores through adjacent row panels. +// stores through adjacent row panels; a ColMajor LHS whose MR-row slices are +// already contiguous per depth step is read from its source (no LHS packing). struct gemm_pack_rhs_first_loop_policy { template <typename Index, typename LhsScalar, typename RhsScalar, typename ResScalar, typename LhsMapper, typename RhsMapper, typename ResMapper, typename PackLhs, typename PackRhs, typename Gebp> @@ -71,8 +140,10 @@ const LhsMapper& lhs, const RhsMapper& rhs, ResMapper& res, PackLhs& pack_lhs, PackRhs& pack_rhs, Gebp& gebp, LhsScalar* blockA, RhsScalar* blockB, ResScalar alpha) { - // Mirror of pack_rhs_once: reuse one full LHS panel across column blocks. + // Mirror of pack_rhs_once: reuse one full LHS panel across column blocks. The in-place read is decided per column + // block, so a block that falls back to packing packs unless an earlier one already did. const bool pack_lhs_once = nc != cols && kc == depth && mc == rows; + bool lhs_packed = false; for (Index j2 = 0; j2 < cols; j2 += nc) { const Index actual_nc = (std::min)(j2 + nc, cols) - j2; @@ -85,8 +156,15 @@ for (Index i2 = 0; i2 < rows; i2 += mc) { const Index actual_mc = (std::min)(i2 + mc, rows) - i2; - - if ((!pack_lhs_once) || j2 == 0) pack_lhs(blockA, lhs.getSubMapper(i2, k2), actual_kc, actual_mc); +#ifdef EIGEN_VECTORIZE_SME + if (sme_run_direct_lhs(bool_constant<sme_direct_lhs_mapper<LhsMapper>::value>(), gebp, + res.getSubMapper(i2, j2), lhs, i2, k2, blockB, actual_mc, actual_kc, actual_nc, alpha)) + continue; +#endif + if (!pack_lhs_once || !lhs_packed) { + pack_lhs(blockA, lhs.getSubMapper(i2, k2), actual_kc, actual_mc); + lhs_packed = true; + } gebp(res.getSubMapper(i2, j2), blockA, blockB, actual_mc, actual_kc, actual_nc, alpha); } } @@ -128,6 +206,13 @@ level3_blocking<LhsScalar, RhsScalar>& blocking, GemmParallelInfo<Index>* info = 0) { // BLAS contract: if alpha == 0, the result is unchanged (and lhs/rhs need not be read). if (numext::is_exactly_zero(alpha)) return; +#ifdef EIGEN_VECTORIZE_SME + // A parallel session needs every thread to take the packed path; scaleAndAddTo runs tiny results on one thread. + if (info == nullptr && sme_run_tiny_gemm<LhsStorageOrder, RhsStorageOrder>( + bool_constant<sme_tiny_gemm_pair<LhsScalar, RhsScalar>::value>(), rows, cols, depth, + lhs_, lhsStride, rhs_, rhsStride, res_, resIncr, resStride, alpha)) + return; +#endif using LhsMapper = const_blas_data_mapper<LhsScalar, Index, LhsStorageOrder>; using RhsMapper = const_blas_data_mapper<RhsScalar, Index, RhsStorageOrder>; @@ -261,6 +346,9 @@ m_blocking.allocateA(); } + // Whether the blocking holds no preallocated buffers, so that disjoint parts of the product can run concurrently. + bool ownsNoBuffers() const { return m_blocking.blockA() == nullptr && m_blocking.blockB() == nullptr; } + void operator()(Index row, Index rows, Index col = 0, Index cols = -1, GemmParallelInfo<Index>* info = 0) const { if (cols == -1) cols = m_rhs.cols(); @@ -396,12 +484,13 @@ m_sizeB = this->m_kc * this->m_nc; } + // The blocking buffers get the temporaries' alignment (64 bytes in SME builds, EIGEN_STACK_ALIGN_BYTES). void allocateA() { - if (this->m_blockA == 0) this->m_blockA = aligned_new<LhsScalar>(m_sizeA); + if (this->m_blockA == 0) this->m_blockA = scratch_new<LhsScalar>(m_sizeA); } void allocateB() { - if (this->m_blockB == 0) this->m_blockB = aligned_new<RhsScalar>(m_sizeB); + if (this->m_blockB == 0) this->m_blockB = scratch_new<RhsScalar>(m_sizeB); } void allocateAll() { @@ -410,8 +499,8 @@ } ~gemm_blocking_space() { - aligned_delete(this->m_blockA, m_sizeA); - aligned_delete(this->m_blockB, m_sizeB); + scratch_delete(this->m_blockA, m_sizeA); + scratch_delete(this->m_blockB, m_sizeB); } }; @@ -468,12 +557,26 @@ } #endif +#ifdef EIGEN_VECTORIZE_SME + // Whether the tiny-result kernel takes a product the coeff-based one would; the GEMM driver sees a RowMajor + // result transposed. + template <typename Dst> + static EIGEN_STRONG_INLINE bool tinyKernelWins(const Dst& dst, const Rhs& rhs) { + constexpr bool row_major = (Dst::Flags & RowMajorBit) != 0; + return sme_tiny_gemm_pair<LhsScalar, RhsScalar>::value && + sme_tiny_gemm_wins<Scalar>(row_major ? dst.cols() : dst.rows(), row_major ? dst.rows() : dst.cols(), + rhs.rows()); + } +#endif + template <typename Dst> static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool useRuntimeCoeffBasedProduct(const Dst& dst, const Rhs& rhs) { if (rhs.rows() <= 0) return false; - if ((rhs.rows() + dst.rows() + dst.cols()) < kCoeffBasedThreshold) return true; #ifdef EIGEN_VECTORIZE_SME - if (outputAreaBelowThreshold(dst)) return true; + if ((rhs.rows() + dst.rows() + dst.cols()) < kCoeffBasedThreshold || outputAreaBelowThreshold(dst)) + return !tinyKernelWins(dst, rhs); +#else + if ((rhs.rows() + dst.rows() + dst.cols()) < kCoeffBasedThreshold) return true; #endif return false; } @@ -556,6 +659,12 @@ ActualLhsTypeCleaned, ActualRhsTypeCleaned, Dest, BlockingType>; BlockingType blocking(dst.rows(), dst.cols(), lhs.cols(), 1, true); +#ifdef EIGEN_VECTORIZE_SME + // A result the tiny-result kernel takes is never worth a parallel session. + if (tinyKernelWins(dst, a_rhs)) + return internal::parallelize_gemm<false>(GemmFunctor(lhs, rhs, dst, actualAlpha, blocking), a_lhs.rows(), + a_rhs.cols(), a_lhs.cols(), Dest::Flags & RowMajorBit); +#endif internal::parallelize_gemm<(Dest::MaxRowsAtCompileTime > 32 || Dest::MaxRowsAtCompileTime == Dynamic)>( GemmFunctor(lhs, rhs, dst, actualAlpha, blocking), a_lhs.rows(), a_rhs.cols(), a_lhs.cols(), Dest::Flags & RowMajorBit);
diff --git a/Eigen/src/Core/products/GeneralMatrixVector.h b/Eigen/src/Core/products/GeneralMatrixVector.h index ef8bc0f..040c5d5 100644 --- a/Eigen/src/Core/products/GeneralMatrixVector.h +++ b/Eigen/src/Core/products/GeneralMatrixVector.h
@@ -53,6 +53,39 @@ using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>; }; +// Whether a mapper's coefficients are addressable and consecutive along the given storage order: the BLAS mappers +// with unit inner stride. Other mappers (e.g. the Tensor contraction input mappers) return coefficients by value. +template <typename Mapper, int Order> +struct gemv_mapper_is_contiguous : std::false_type {}; +template <typename Scalar, typename Index, int Order, int Alignment> +struct gemv_mapper_is_contiguous<blas_data_mapper<Scalar, Index, Order, Alignment, 1>, Order> : std::true_type {}; +template <typename Scalar, typename Index, int Order> +struct gemv_mapper_is_contiguous<const_blas_data_mapper<Scalar, Index, Order>, Order> : std::true_type {}; + +// Whether the GEMV kernels finish a partial packet with masked segment loads. Segment loads zero the lanes outside +// the segment, which the row-major kernel relies on before its horizontal reduction. +template <typename LhsPacket, typename RhsPacket, typename ResPacket> +using gemv_use_packet_segment = + bool_constant<has_packet_segment<ResPacket>::value && std::is_same<LhsPacket, ResPacket>::value && + std::is_same<RhsPacket, ResPacket>::value>; + +// Loads count < packet-size consecutive coefficients starting at (i, j), along Order. Only contiguous mappers take +// the masked load; the kernels never select segments for other mappers, whose branch merely has to compile. +template <typename Packet, int Order, typename Mapper, + bool Contiguous = gemv_mapper_is_contiguous<Mapper, Order>::value> +struct gemv_segment_loader { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Mapper&, Index, Index, Index) { + eigen_internal_assert(false && "segment load from a non-contiguous mapper"); + return pzero(Packet{}); + } +}; +template <typename Packet, int Order, typename Mapper> +struct gemv_segment_loader<Packet, Order, Mapper, true> { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Mapper& m, Index i, Index j, Index count) { + return ploaduSegment<Packet>(&m(i, j), 0, count); + } +}; + /* Optimized col-major matrix * vector product: * This algorithm processes the matrix per vertical panels, * which are then processed horizontally per chunk of 8*PacketSize x 1 vertical segments. @@ -91,10 +124,24 @@ EIGEN_DEVICE_FUNC inline static void run(Index rows, Index cols, const LhsMapper& lhs, const RhsMapper& rhs, ResScalar* res, Index resIncr, RhsScalar alpha); - template <int N> + // Processes N full packets of rows starting at row i and, if Segment, the next count < ResPacketSize rows as one + // masked packet, all in a single pass over the columns [j2, jend). + template <int N, bool Segment = false> EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE void process_rows( Index i, Index j2, Index jend, const LhsMapper& lhs, const RhsMapper& rhs, ResScalar* res, - const ResPacket& palpha, conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>& pcj); + const ResPacket& palpha, conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>& pcj, Index count = 0); + + // Finishes the rows from i on, full_packets < 10 full packets followed by 0 < count < ResPacketSize rows, in one + // process_rows pass. Out of line: it runs at most once per column block, and only when rows is not a multiple of + // the packet size. lhs is taken by value: a reference would make run() keep its copy in memory, which clang fills + // with a load that cannot be forwarded from the caller's stores. + EIGEN_DEVICE_FUNC static EIGEN_DONT_INLINE void process_segment_tail( + std::true_type, Index full_packets, Index i, Index j2, Index jend, LhsMapper lhs, const RhsMapper& rhs, + ResScalar* res, const ResPacket& palpha, conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>& pcj, + Index count); + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void process_segment_tail( + std::false_type, Index, Index, Index, Index, const LhsMapper&, const RhsMapper&, ResScalar*, const ResPacket&, + conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>&, Index) {} }; // Integer-sequence helper for col-major GEMV full-packet row blocks. @@ -149,24 +196,79 @@ } }; +// No full packets: the final pass may hold only the masked partial packet. +template <> +struct gemv_colmajor_unroller<0> { + template <typename Packet> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void init_zero(Packet*) {} + + template <typename LhsPacket, int LhsStride, int Alignment, typename AccPacket, typename RhsPacket, + typename ConjHelper, typename LhsMapper, typename Index> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void madd(AccPacket*, const LhsMapper&, Index, Index, const RhsPacket&, + ConjHelper&) {} + + template <typename ResPacket, int ResStride, typename ResScalar, typename Index> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void store(const ResPacket*, ResScalar*, Index, const ResPacket&) {} +}; + template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar, typename RhsMapper, bool ConjugateRhs, int Version> -template <int N> -EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void general_matrix_vector_product< - Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs, - Version>::process_rows(Index i, Index j2, Index jend, const LhsMapper& lhs, const RhsMapper& rhs, ResScalar* res, - const ResPacket& palpha, - conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>& pcj) { +template <int N, bool Segment> +EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void +general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs, + Version>::process_rows(Index i, Index j2, Index jend, const LhsMapper& lhs, + const RhsMapper& rhs, ResScalar* res, const ResPacket& palpha, + conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>& pcj, + Index count) { enum { LhsAlignment = Unaligned, LhsPacketSize = Traits::LhsPacketSize, ResPacketSize = Traits::ResPacketSize }; using Unroller = gemv_colmajor_unroller<N>; + const Index iseg = i + N * ResPacketSize; - ResPacket c[N]; + // c_seg accumulates the masked partial packet when Segment is set. It is not part of c: a wider array grows the + // stack frame GCC estimates for run() and stops it from being inlined. + ResPacket c[N > 0 ? N : 1]; Unroller::init_zero(c); + ResPacket c_seg = pzero(ResPacket{}); for (Index j = j2; j < jend; ++j) { RhsPacket b0 = pset1<RhsPacket>(rhs(j, 0)); Unroller::template madd<LhsPacket, LhsPacketSize, LhsAlignment>(c, lhs, i, j, b0, pcj); + EIGEN_IF_CONSTEXPR (Segment) { + c_seg = pcj.pmadd(gemv_segment_loader<LhsPacket, ColMajor, LhsMapper>::run(lhs, iseg, j, count), b0, c_seg); + } } Unroller::template store<ResPacket, ResPacketSize>(c, res, i, palpha); + EIGEN_IF_CONSTEXPR (Segment) { + pstoreuSegment(res + iseg, pmadd(c_seg, palpha, ploaduSegment<ResPacket>(res + iseg, 0, count)), 0, count); + } +} + +template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar, + typename RhsMapper, bool ConjugateRhs, int Version> +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void general_matrix_vector_product< + Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs, + Version>::process_segment_tail(std::true_type, Index full_packets, Index i, Index j2, Index jend, LhsMapper lhs, + const RhsMapper& rhs, ResScalar* res, const ResPacket& palpha, + conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>& pcj, Index count) { +#define EIGEN_GEMV_PROCESS_ROW(n) \ + case n: \ + process_rows<n, true>(i, j2, jend, lhs, rhs, res, palpha, pcj, count); \ + break + switch (full_packets) { + EIGEN_GEMV_PROCESS_ROW(0); + EIGEN_GEMV_PROCESS_ROW(1); + EIGEN_GEMV_PROCESS_ROW(2); + EIGEN_GEMV_PROCESS_ROW(3); + EIGEN_GEMV_PROCESS_ROW(4); + EIGEN_GEMV_PROCESS_ROW(5); + EIGEN_GEMV_PROCESS_ROW(6); + EIGEN_GEMV_PROCESS_ROW(7); + EIGEN_GEMV_PROCESS_ROW(8); + EIGEN_GEMV_PROCESS_ROW(9); + default: + eigen_internal_assert(false); + break; + } +#undef EIGEN_GEMV_PROCESS_ROW } template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar, @@ -201,10 +303,18 @@ ResPacketSizeQuarter = QuarterTraits::ResPacketSize, LhsPacketSize = Traits::LhsPacketSize, HasHalf = (int)ResPacketSizeHalf < (int)ResPacketSize, - HasQuarter = (int)ResPacketSizeQuarter < (int)ResPacketSizeHalf + HasQuarter = (int)ResPacketSizeQuarter < (int)ResPacketSizeHalf, + UseSegment = gemv_use_packet_segment<LhsPacket, RhsPacket, ResPacket>::value && + gemv_mapper_is_contiguous<LhsMapper, ColMajor>::value }; - const Index n8 = rows - 8 * ResPacketSize + 1; + using UnsignedIndex = std::make_unsigned_t<Index>; + // With segments, the count trailing rows of a partial packet take one masked pass together with the full packets + // before them. The 8-packet blocks then stop while at least 2 full packets remain, so that pass is never a lone + // packet whose accumulator chain is bound by the FMA latency. Exact multiples of the packet size keep the 4, 3, 2 + // and 1-packet passes. + const Index count = UseSegment ? Index(UnsignedIndex(rows) % ResPacketSize) : Index(0); + const Index n8 = rows - (count > 0 ? 10 : 8) * ResPacketSize + 1; const Index n4 = rows - 4 * ResPacketSize + 1; const Index n3 = rows - 3 * ResPacketSize + 1; const Index n2 = rows - 2 * ResPacketSize + 1; @@ -226,50 +336,50 @@ Index jend = numext::mini(j2 + block_cols, cols); Index i = 0; for (; i < n8; i += ResPacketSize * 8) process_rows<8>(i, j2, jend, lhs, rhs, res, palpha, pcj); - if (i < n4) { - process_rows<4>(i, j2, jend, lhs, rhs, res, palpha, pcj); - i += ResPacketSize * 4; - } - if (i < n3) { - process_rows<3>(i, j2, jend, lhs, rhs, res, palpha, pcj); - i += ResPacketSize * 3; - } - if (i < n2) { - process_rows<2>(i, j2, jend, lhs, rhs, res, palpha, pcj); - i += ResPacketSize * 2; - } - if (i < n1) { - process_rows<1>(i, j2, jend, lhs, rhs, res, palpha, pcj); - i += ResPacketSize; - } - EIGEN_IF_CONSTEXPR (HasHalf) { - if (i < n_half) { - ResPacketHalf c0 = pzero(ResPacketHalf{}); - for (Index j = j2; j < jend; j += 1) { - RhsPacketHalf b0 = pset1<RhsPacketHalf>(rhs(j, 0)); - c0 = pcj_half.pmadd(lhs.template load<LhsPacketHalf, LhsAlignment>(i + 0, j), b0, c0); + if (count > 0) { + process_segment_tail(bool_constant<UseSegment>(), Index(UnsignedIndex(rows - i) / ResPacketSize), i, j2, jend, + lhs, rhs, res, palpha, pcj, count); + } else { +#define EIGEN_GEMV_PROCESS_ROW(k) \ + if (i < n##k) { \ + process_rows<k>(i, j2, jend, lhs, rhs, res, palpha, pcj); \ + i += ResPacketSize * (k); \ + } \ + static_assert(true, "Trailing semicolon required") + EIGEN_GEMV_PROCESS_ROW(4); + EIGEN_GEMV_PROCESS_ROW(3); + EIGEN_GEMV_PROCESS_ROW(2); + EIGEN_GEMV_PROCESS_ROW(1); +#undef EIGEN_GEMV_PROCESS_ROW + EIGEN_IF_CONSTEXPR (HasHalf) { + if (i < n_half) { + ResPacketHalf c0 = pzero(ResPacketHalf{}); + for (Index j = j2; j < jend; j += 1) { + RhsPacketHalf b0 = pset1<RhsPacketHalf>(rhs(j, 0)); + c0 = pcj_half.pmadd(lhs.template load<LhsPacketHalf, LhsAlignment>(i + 0, j), b0, c0); + } + pstoreu(res + i + ResPacketSizeHalf * 0, + pmadd(c0, palpha_half, ploadu<ResPacketHalf>(res + i + ResPacketSizeHalf * 0))); + i += ResPacketSizeHalf; } - pstoreu(res + i + ResPacketSizeHalf * 0, - pmadd(c0, palpha_half, ploadu<ResPacketHalf>(res + i + ResPacketSizeHalf * 0))); - i += ResPacketSizeHalf; } - } - EIGEN_IF_CONSTEXPR (HasQuarter) { - if (i < n_quarter) { - ResPacketQuarter c0 = pzero(ResPacketQuarter{}); - for (Index j = j2; j < jend; j += 1) { - RhsPacketQuarter b0 = pset1<RhsPacketQuarter>(rhs(j, 0)); - c0 = pcj_quarter.pmadd(lhs.template load<LhsPacketQuarter, LhsAlignment>(i + 0, j), b0, c0); + EIGEN_IF_CONSTEXPR (HasQuarter) { + if (i < n_quarter) { + ResPacketQuarter c0 = pzero(ResPacketQuarter{}); + for (Index j = j2; j < jend; j += 1) { + RhsPacketQuarter b0 = pset1<RhsPacketQuarter>(rhs(j, 0)); + c0 = pcj_quarter.pmadd(lhs.template load<LhsPacketQuarter, LhsAlignment>(i + 0, j), b0, c0); + } + pstoreu(res + i + ResPacketSizeQuarter * 0, + pmadd(c0, palpha_quarter, ploadu<ResPacketQuarter>(res + i + ResPacketSizeQuarter * 0))); + i += ResPacketSizeQuarter; } - pstoreu(res + i + ResPacketSizeQuarter * 0, - pmadd(c0, palpha_quarter, ploadu<ResPacketQuarter>(res + i + ResPacketSizeQuarter * 0))); - i += ResPacketSizeQuarter; } - } - for (; i < rows; ++i) { - ResScalar c0(0); - for (Index j = j2; j < jend; j += 1) c0 += cj.pmul(lhs(i, j), rhs(j, 0)); - res[i] += alpha * c0; + for (; i < rows; ++i) { + ResScalar c0(0); + for (Index j = j2; j < jend; j += 1) c0 += cj.pmul(lhs(i, j), rhs(j, 0)); + res[i] += alpha * c0; + } } } } @@ -343,11 +453,16 @@ ? (int)QuarterTraits::LhsPacketSize : (((int)HalfTraits::LhsPacketSize < (int)Traits::LhsPacketSize) ? (int)HalfTraits::LhsPacketSize : (int)Traits::LhsPacketSize), - HasSubPackets_ = (int)MinUsefulCols_ < (int)LhsPacketSize_ + HasSubPackets_ = (int)MinUsefulCols_ < (int)LhsPacketSize_, + UseSegment_ = gemv_use_packet_segment<LhsPacket, RhsPacket, ResPacket>::value && + gemv_mapper_is_contiguous<LhsMapper, RowMajor>::value && + gemv_mapper_is_contiguous<RhsMapper, ColMajor>::value, + // With segments, one masked full packet per row beats a half packet followed by two or more scalar columns. + SmallColsEnd_ = UseSegment_ ? (int)HalfTraits::LhsPacketSize + 2 : (int)LhsPacketSize_ }; EIGEN_IF_CONSTEXPR (HasSubPackets_) { if (cols >= MinUsefulCols_) { - if (cols < LhsPacketSize_) { + if (cols < SmallColsEnd_ && cols < LhsPacketSize_) { run_small_cols(rows, cols, alhs, rhs, res, resIncr, alpha); return; } @@ -384,13 +499,20 @@ LhsPacketSizeHalf = HalfTraits::LhsPacketSize, LhsPacketSizeQuarter = QuarterTraits::LhsPacketSize, HasHalf = (int)ResPacketSizeHalf < (int)ResPacketSize, - HasQuarter = (int)ResPacketSizeQuarter < (int)ResPacketSizeHalf + HasQuarter = (int)ResPacketSizeQuarter < (int)ResPacketSizeHalf, + UseSegment = UseSegment_ }; using UnsignedIndex = std::make_unsigned_t<Index>; const Index fullColBlockEnd = LhsPacketSize * (UnsignedIndex(cols) / LhsPacketSize); const Index halfColBlockEnd = LhsPacketSizeHalf * (UnsignedIndex(cols) / LhsPacketSizeHalf); const Index quarterColBlockEnd = LhsPacketSizeQuarter * (UnsignedIndex(cols) / LhsPacketSizeQuarter); + // With segments, the last cols - fullColBlockEnd < LhsPacketSize columns take one masked packet per row and the + // scalar column loops below are empty. + const Index segmentCount = cols - fullColBlockEnd; + const Index scalarColStart = UseSegment ? cols : fullColBlockEnd; + using LhsSegmentLoader = gemv_segment_loader<LhsPacket, RowMajor, LhsMapper>; + using RhsSegmentLoader = gemv_segment_loader<RhsPacket, ColMajor, RhsMapper>; Index i = 0; for (; i < n8; i += 8) { @@ -409,6 +531,17 @@ c6 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i + 6, j), b0, c6); c7 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i + 7, j), b0, c7); } + if (UseSegment && segmentCount > 0) { + RhsPacket b0 = RhsSegmentLoader::run(rhs, fullColBlockEnd, 0, segmentCount); + c0 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 0, fullColBlockEnd, segmentCount), b0, c0); + c1 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 1, fullColBlockEnd, segmentCount), b0, c1); + c2 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 2, fullColBlockEnd, segmentCount), b0, c2); + c3 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 3, fullColBlockEnd, segmentCount), b0, c3); + c4 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 4, fullColBlockEnd, segmentCount), b0, c4); + c5 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 5, fullColBlockEnd, segmentCount), b0, c5); + c6 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 6, fullColBlockEnd, segmentCount), b0, c6); + c7 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 7, fullColBlockEnd, segmentCount), b0, c7); + } ResScalar cc0 = predux(c0); ResScalar cc1 = predux(c1); ResScalar cc2 = predux(c2); @@ -418,7 +551,7 @@ ResScalar cc6 = predux(c6); ResScalar cc7 = predux(c7); - for (Index j = fullColBlockEnd; j < cols; ++j) { + for (Index j = scalarColStart; j < cols; ++j) { RhsScalar b0 = rhs(j, 0); cc0 += cj.pmul(lhs(i + 0, j), b0); @@ -450,12 +583,19 @@ c2 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i + 2, j), b0, c2); c3 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i + 3, j), b0, c3); } + if (UseSegment && segmentCount > 0) { + RhsPacket b0 = RhsSegmentLoader::run(rhs, fullColBlockEnd, 0, segmentCount); + c0 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 0, fullColBlockEnd, segmentCount), b0, c0); + c1 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 1, fullColBlockEnd, segmentCount), b0, c1); + c2 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 2, fullColBlockEnd, segmentCount), b0, c2); + c3 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 3, fullColBlockEnd, segmentCount), b0, c3); + } ResScalar cc0 = predux(c0); ResScalar cc1 = predux(c1); ResScalar cc2 = predux(c2); ResScalar cc3 = predux(c3); - for (Index j = fullColBlockEnd; j < cols; ++j) { + for (Index j = scalarColStart; j < cols; ++j) { RhsScalar b0 = rhs(j, 0); cc0 += cj.pmul(lhs(i + 0, j), b0); @@ -477,10 +617,15 @@ c0 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i + 0, j), b0, c0); c1 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i + 1, j), b0, c1); } + if (UseSegment && segmentCount > 0) { + RhsPacket b0 = RhsSegmentLoader::run(rhs, fullColBlockEnd, 0, segmentCount); + c0 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 0, fullColBlockEnd, segmentCount), b0, c0); + c1 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 1, fullColBlockEnd, segmentCount), b0, c1); + } ResScalar cc0 = predux(c0); ResScalar cc1 = predux(c1); - for (Index j = fullColBlockEnd; j < cols; ++j) { + for (Index j = scalarColStart; j < cols; ++j) { RhsScalar b0 = rhs(j, 0); cc0 += cj.pmul(lhs(i + 0, j), b0); @@ -498,22 +643,26 @@ RhsPacket b0 = rhs.template load<RhsPacket, Unaligned>(j, 0); c0 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i, j), b0, c0); } + if (UseSegment && segmentCount > 0) { + RhsPacket b0 = RhsSegmentLoader::run(rhs, fullColBlockEnd, 0, segmentCount); + c0 = pcj.pmadd(LhsSegmentLoader::run(lhs, i, fullColBlockEnd, segmentCount), b0, c0); + } ResScalar cc0 = predux(c0); - EIGEN_IF_CONSTEXPR (HasHalf) { + EIGEN_IF_CONSTEXPR (HasHalf && !UseSegment) { for (Index j = fullColBlockEnd; j < halfColBlockEnd; j += LhsPacketSizeHalf) { RhsPacketHalf b0 = rhs.template load<RhsPacketHalf, Unaligned>(j, 0); c0_h = pcj_half.pmadd(lhs.template load<LhsPacketHalf, LhsAlignment>(i, j), b0, c0_h); } cc0 += predux(c0_h); } - EIGEN_IF_CONSTEXPR (HasQuarter) { + EIGEN_IF_CONSTEXPR (HasQuarter && !UseSegment) { for (Index j = halfColBlockEnd; j < quarterColBlockEnd; j += LhsPacketSizeQuarter) { RhsPacketQuarter b0 = rhs.template load<RhsPacketQuarter, Unaligned>(j, 0); c0_q = pcj_quarter.pmadd(lhs.template load<LhsPacketQuarter, LhsAlignment>(i, j), b0, c0_q); } cc0 += predux(c0_q); } - for (Index j = quarterColBlockEnd; j < cols; ++j) { + for (Index j = UseSegment ? cols : quarterColBlockEnd; j < cols; ++j) { cc0 += cj.pmul(lhs(i, j), rhs(j, 0)); } res[i * resIncr] += alpha * cc0;
diff --git a/Eigen/src/Core/products/Parallelizer.h b/Eigen/src/Core/products/Parallelizer.h index 1d82a11..0220e05 100644 --- a/Eigen/src/Core/products/Parallelizer.h +++ b/Eigen/src/Core/products/Parallelizer.h
@@ -62,6 +62,72 @@ * \sa nbThreads */ inline void setNbThreads(int v) { internal::manage_multi_threading(SetAction, &v); } +#ifdef EIGEN_VECTORIZE_SME +#ifndef EIGEN_SME_MIN_TASK_SIZE +#define EIGEN_SME_MIN_TASK_SIZE (double(1 << 20)) +#endif +namespace internal { +// Time of one multiply-add on the SME kernel in fp32 multiply-adds: an FP64 outer product covers a quarter of the +// elements of an FP32 one at the same vector length, and a complex multiply-add is four real ones. +template <typename Scalar> +struct sme_madd_cost { + static constexpr double value = (std::is_same<typename NumTraits<Scalar>::Real, double>::value ? 4.0 : 1.0) * + (NumTraits<Scalar>::IsComplex ? 4.0 : 1.0); +}; +// Apple silicon shares one SME unit per core cluster, so more threads than clusters only contend for the units; other +// SME implementations may have one per core. The count comes from macOS's performance-cluster topology and is 0 (no +// cap) everywhere else, where EIGEN_SME_UNITS or setNbSmeUnits() supply it. +inline int detect_sme_units() { +#if defined(EIGEN_SME_UNITS) + return EIGEN_SME_UNITS; +#elif EIGEN_OS_MAC + // Performance clusters only (a cluster shares one L2): the work is split evenly over the threads, + // so a thread on the efficiency cluster's slower unit would set the finishing time. + int32_t cores = 0, per_l2 = 0; + size_t sz = sizeof(cores); + if (sysctlbyname("hw.perflevel0.physicalcpu", &cores, &sz, nullptr, 0) != 0 || cores <= 0) return 0; + sz = sizeof(per_l2); + if (sysctlbyname("hw.perflevel0.cpusperl2", &per_l2, &sz, nullptr, 0) != 0 || per_l2 <= 0) return 0; + return numext::maxi<int32_t>(1, cores / per_l2); +#else + return 0; +#endif +} +inline void manage_sme_units(Action action, int* v) { + static int m_units = detect_sme_units(); + if (action == SetAction) + m_units = *v; + else + *v = m_units; +} +} // namespace internal +#endif + +/** \returns the number of ARM SME units a product on the SME GEMM kernel spreads over, or 0 when it is unknown or + * the SME backend is not in use (then \c nbThreads() applies). Detected once from the core topology. + * \sa setNbSmeUnits */ +inline int nbSmeUnits() { +#ifdef EIGEN_VECTORIZE_SME + int ret; + internal::manage_sme_units(GetAction, &ret); + return ret; +#else + return 0; +#endif +} +/** Sets the number of SME units, which caps the threads a product on the SME GEMM kernel uses and, when nonzero, gives + * each of them a disjoint part of the result instead of a shared parallel session; 0 removes both. + * \c EIGEN_SME_UNITS sets it at compile time. Does nothing when the SME backend is not in use. Like setNbThreads(), + * it must not be called while a product runs on another thread. + * \sa nbSmeUnits */ +inline void setNbSmeUnits(int v) { +#ifdef EIGEN_VECTORIZE_SME + internal::manage_sme_units(SetAction, &v); +#else + EIGEN_UNUSED_VARIABLE(v); +#endif +} + #ifdef EIGEN_GEMM_THREADPOOL // Sets the ThreadPool used by Eigen parallel Gemm. // @@ -210,10 +276,33 @@ // compute the maximal number of threads from the total amount of work: double work = static_cast<double>(rows) * static_cast<double>(cols) * static_cast<double>(depth); double kMinTaskSize = 50000; // FIXME: tune this minimum task-size heuristic based on architecture and scalar type. +#ifdef EIGEN_VECTORIZE_SME + // With a known SME unit count (one unit per core cluster), the SME kernel does far more work per unit of time than + // the generic one, so a thread needs more of it to repay the handoff; elsewhere the size is not measured and the + // generic one applies. With no preallocated buffers, each thread runs a disjoint part of the result (below). + constexpr bool kSme = + sme_has_gebp_kernel<typename Functor::Traits::LhsScalar, typename Functor::Traits::RhsScalar>::value; + const bool sme_units_known = kSme && nbSmeUnits() > 0; + const bool sme_disjoint = sme_units_known && func.ownsNoBuffers(); + if (sme_units_known) + kMinTaskSize = EIGEN_SME_MIN_TASK_SIZE / sme_madd_cost<typename Functor::Traits::LhsScalar>::value; + if (sme_disjoint) { + const Index kernel_rows = transpose ? cols : rows, kernel_cols = transpose ? rows : cols; + pb_max_threads = + std::max<Index>(1, (std::max)(kernel_rows / Functor::Traits::mr, kernel_cols / Functor::Traits::nr)); + } +#endif pb_max_threads = std::max<Index>(1, std::min<Index>(pb_max_threads, static_cast<Index>(work / kMinTaskSize))); // compute the number of threads we are going to use int threads = std::min<int>(nbThreads(), static_cast<int>(pb_max_threads)); +#ifdef EIGEN_VECTORIZE_SME + // The SME kernels share one unit per cluster; more threads than units only contend. + EIGEN_IF_CONSTEXPR (kSme) { + const int units = nbSmeUnits(); + if (units > 0) threads = std::min<int>(threads, units); + } +#endif // if multi-threading is explicitly disabled, not useful, or if we already are // inside a parallel session, then abort multi-threading @@ -231,6 +320,68 @@ #endif if (dont_parallelize) return func(0, rows, 0, cols); +#ifdef EIGEN_VECTORIZE_SME + // A known unit count stands for one SME unit per core cluster, each with its own L2 (see nbSmeUnits): the + // cooperative session below would read the other cluster's packed LHS and wait for it at every depth step, so each + // thread runs a disjoint part of the result instead. + EIGEN_IF_CONSTEXPR (kSme) { + if (sme_disjoint) { + // On the kernel's problem (the transposed one for a row-major result), a row split repacks the whole RHS per + // thread and a column split the whole LHS (a copy, or nothing when read in place): split the rows only when + // they are twice the columns and give 8 LHS panels per part, since a narrower result already runs close to peak. + const Index kernel_rows = transpose ? cols : rows, kernel_cols = transpose ? rows : cols; + const Index row_parts = kernel_rows / (8 * Functor::Traits::mr); + const bool split_kernel_rows = kernel_rows >= 2 * kernel_cols && row_parts >= 2; + const bool split_rows = transpose ? !split_kernel_rows : split_kernel_rows; + if (split_kernel_rows) threads = static_cast<int>((std::min)(Index(threads), row_parts)); + // The result's rows are the kernel's columns for a row-major result. No part may be empty. + const Index row_grain = transpose ? Functor::Traits::nr : Functor::Traits::mr; + const Index col_grain = transpose ? Functor::Traits::mr : Functor::Traits::nr; + const Index chunks = split_rows ? (rows + row_grain - 1) / row_grain : (cols + col_grain - 1) / col_grain; + threads = static_cast<int>((std::min)(Index(threads), chunks)); + if (threads <= 1) return func(0, rows, 0, cols); + auto part = [&func, rows, cols, threads, split_rows, row_grain, col_grain](int i) { + Index start, length; + if (split_rows) { + balanced_gemm_range<Index>(rows, threads, row_grain, i, start, length); + if (length > 0) func(start, length, 0, cols); + } else { + balanced_gemm_range<Index>(cols, threads, col_grain, i, start, length); + if (length > 0) func(0, rows, start, length); + } + }; +#if defined(EIGEN_HAS_OPENMP) +#pragma omp parallel for num_threads(threads) schedule(static, 1) + for (int i = 0; i < threads; ++i) part(i); +#elif defined(EIGEN_GEMM_THREADPOOL) + // The parts take about as long as each other, so the caller spins for the others (tens of microseconds, less than + // a blocking wake-up costs a short product) before it blocks on the barrier, which it always passes, also + // before an exception from its own part leaves this frame. + std::atomic<int> pending(threads - 1); + Barrier done(threads - 1); + for (int i = 0; i < threads - 1; ++i) + pool->Schedule([&part, &pending, &done, i] { + part(i); + pending.fetch_sub(1, std::memory_order_release); + done.Notify(); + }); + const auto wait = [&pending, &done] { + for (int spin = 0; spin < 65536 && pending.load(std::memory_order_acquire) != 0; ++spin) { + } + done.Wait(); + }; + EIGEN_TRY { part(threads - 1); } + EIGEN_CATCH(...) { + wait(); + EIGEN_THROW; + } + wait(); +#endif + return; + } + } +#endif + func.initParallelSession(threads); if (transpose) std::swap(rows, cols);
diff --git a/Eigen/src/Core/products/TriangularSolverMatrix.h b/Eigen/src/Core/products/TriangularSolverMatrix.h index 03fbe14..59454b6 100644 --- a/Eigen/src/Core/products/TriangularSolverMatrix.h +++ b/Eigen/src/Core/products/TriangularSolverMatrix.h
@@ -43,8 +43,9 @@ Index otherStride); }; -// The packet lanes are independent right-hand sides. Reserve registers for the -// RHS packets and coefficient broadcasts in addition to the output accumulators. +// The packet lanes are independent right-hand sides. Half the registers hold output accumulators, +// enough independent multiply-add chains to cover the FMA latency on 16- and 32-register targets; +// the rest hold the RHS packets and coefficient broadcasts. template <typename Scalar> struct triangular_solve_packet_traits { static constexpr bool Enabled = packet_traits<Scalar>::Vectorizable && @@ -52,8 +53,12 @@ std::numeric_limits<Scalar>::is_iec559 && std::numeric_limits<Scalar>::radix == 2; static constexpr int PacketSize = packet_traits<Scalar>::size; static constexpr int RegisterRows = 4; - static constexpr int RhsPackets = - plain_enum_max(1, plain_enum_min(2, (gebp_traits<Scalar, Scalar>::NumberOfRegisters - 2) / (RegisterRows + 1))); + static constexpr int NumberOfRegisters = gebp_traits<Scalar, Scalar>::NumberOfRegisters; + static constexpr int RhsPackets = plain_enum_max( + 1, plain_enum_min((NumberOfRegisters - 2) / (RegisterRows + 1), NumberOfRegisters / (2 * RegisterRows))); + // Rows solved per step for a tile of the given RHS packets: a narrow tile takes twice the rows while + // the extra accumulators fit in a full tile's registers, keeping more multiply-add chains in flight. + static constexpr int tile_rows(int packets) { return packets * 2 <= RhsPackets ? 2 * RegisterRows : RegisterRows; } // Allocation bound, independent of the cache-dependent direct-solve cutoff. static constexpr int WorkspaceRows = 128; #if defined(EIGEN_VECTORIZE_AVX512) && EIGEN_USE_AVX512_TRSM_L_KERNELS @@ -141,8 +146,7 @@ } // The braced expansion orders the dependent solves by row. int solve_rows[] = { - 0, - (solve_row<Rows>(x, a, inverse, r0, step, std::make_index_sequence<Traits::RegisterRows - Rows - 1>{}), 0)...}; + 0, (solve_row<Rows>(x, a, inverse, r0, step, std::make_index_sequence<sizeof...(Rows) - Rows - 1>{}), 0)...}; EIGEN_UNUSED_VARIABLE(solve_rows); int store_rows[] = {0, (work[r0 + Index(Rows) * step] = x[Rows], 0)...}; EIGEN_UNUSED_VARIABLE(store_rows); @@ -166,6 +170,11 @@ } Index i = 0; const Index step = IsLower ? 1 : -1; + constexpr int TileRows = Traits::tile_rows(RhsPackets); + EIGEN_IF_CONSTEXPR (TileRows != Traits::RegisterRows) { + for (; i + TileRows <= size; i += TileRows) + solve_block(i, a, inverse, work, IsLower ? i : size - i - 1, step, std::make_index_sequence<TileRows>{}); + } for (; i + Traits::RegisterRows <= size; i += Traits::RegisterRows) { const Index r0 = IsLower ? i : size - i - 1; // Preserve the four-row schedule that keeps GCC and Clang's hot loops compact. @@ -233,9 +242,25 @@ } } + // The last cols < PacketSize columns, zero-padded to a packet in a copy. For a row-major triangle this costs + // one packet solve instead of a scalar dot product per coefficient; a column-major triangle's scalar solve + // is a contiguous axpy per column, which compilers vectorize. Padded lanes are dropped. Inlined, its buffer + // and second solve<1> copy measured slower even for solves that never pad. + static EIGEN_DONT_INLINE void solve_padded(Index size, const TriMapper& a, const Scalar* inverse, Scalar* other, + Index otherStride, Index cols) { + Map<Matrix<Scalar, Dynamic, Dynamic, ColMajor>, Unaligned, OuterStride<>> rest(other, size, cols, + OuterStride<>(otherStride)); + Matrix<Scalar, Dynamic, Dynamic, ColMajor, Traits::WorkspaceRows, PacketSize> padded(size, Index(PacketSize)); + padded.leftCols(cols) = rest; + padded.rightCols(PacketSize - cols).setZero(); + solve<1>(size, a, inverse, padded.data(), size); + rest = padded.leftCols(cols); + } + static EIGEN_DONT_INLINE void kernel(Index size, Index cols, const Scalar* tri, Index triStride, Scalar* other, Index otherStride) { - eigen_internal_assert(size <= Traits::WorkspaceRows && cols % PacketSize == 0); + eigen_internal_assert(size <= Traits::WorkspaceRows && cols >= PacketSize && + (TriStorageOrder == RowMajor || cols % PacketSize == 0)); EIGEN_IF_CONSTEXPR (!IsLower) { tri -= (size - 1) * (triStride + 1); other -= size - 1; @@ -255,7 +280,14 @@ for (; j + Traits::RhsPackets * PacketSize <= cols; j += Traits::RhsPackets * PacketSize) solve<Traits::RhsPackets>(size, a, inverse, other + j * otherStride, otherStride); } + EIGEN_IF_CONSTEXPR (Traits::RhsPackets > 2) { + for (; j + 2 * PacketSize <= cols; j += 2 * PacketSize) + solve<2>(size, a, inverse, other + j * otherStride, otherStride); + } for (; j + PacketSize <= cols; j += PacketSize) solve<1>(size, a, inverse, other + j * otherStride, otherStride); + EIGEN_IF_CONSTEXPR (TriStorageOrder == RowMajor) { + if (j < cols) solve_padded(size, a, inverse, other + j * otherStride, otherStride, cols - j); + } } }; @@ -268,7 +300,9 @@ EIGEN_IF_CONSTEXPR ((Specialized && OtherInnerStride == 1 && triangular_solve_packet_traits<Scalar>::Enabled)) { if (size >= triangular_solve_packet_traits<Scalar>::RegisterRows && size <= triangular_solve_packet_traits<Scalar>::WorkspaceRows && otherSize >= packet_traits<Scalar>::size) { - const Index packetCols = numext::round_down(otherSize, Index(packet_traits<Scalar>::size)); + // The packet kernel pads the last columns only for a row-major triangle; see solve_padded. + const Index packetCols = + TriStorageOrder == RowMajor ? otherSize : numext::round_down(otherSize, Index(packet_traits<Scalar>::size)); triangular_solve_packet_kernel<Scalar, Index, Mode, TriStorageOrder>::kernel(size, packetCols, _tri, triStride, _other, otherStride); if (packetCols == otherSize) return; @@ -515,7 +549,8 @@ const std::ptrdiff_t budget = triangular_solve_budget<Scalar>(l2, l3); const Index nc = triangular_solve_panel_columns(size, otherSize, budget, Index(Traits::nr)); const Index mc = (numext::mini)(size, blocking.mc()); // cache block size along the M direction - // The tr solve below packs up to SmallPanelWidth x kc entries of the triangle into blockA. + // The tr solve below packs up to panelWidth x kc entries of the triangle into blockA, where panelWidth is + // SmallPanelWidth or, with the packet kernel, at most blockARows. const Index blockARows = (numext::maxi)(mc, Index(SmallPanelWidth)); // cache block size along the K direction const Index kc = triangular_solve_kc<Scalar>(size, otherSize, (numext::maxi)(blockARows, nc), budget, @@ -536,7 +571,25 @@ // the goal here is to subdivide the Rhs panels such that we keep some cache // coherence when accessing the rhs elements Index subcols = otherSize > 0 ? l2 / (4 * sizeof(Scalar) * numext::maxi<Index>(otherStride, size)) : 0; - subcols = numext::maxi<Index>((subcols / Traits::nr) * Traits::nr, Traits::nr); + Index colStep = Traits::nr; + // Width of the diagonal panels solved between gebp updates of their depth. + Index panelWidth = SmallPanelWidth; + EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 && triangular_solve_packet_traits<Scalar>::UseUnblocked)) { + // The packet kernel solves a whole diagonal block faster than narrow panels do, since those + // interleave slower rank-SmallPanelWidth gebp updates; a block deeper than its workspace is split + // evenly. That needs a tile of two packets of right-hand sides where the registers allow: a single + // packet measured slower. pack_rhs reads each kc x subcols chunk right after its solve, so a chunk should + // stay in L2; packet-aligned chunks confine any column tail to an Rhs panel's last chunk. + using PacketTraits = triangular_solve_packet_traits<Scalar>; + const Index rows = Index(PacketTraits::RegisterRows), packet = Index(PacketTraits::PacketSize); + if (otherSize >= Index(plain_enum_min(2, PacketTraits::RhsPackets)) * packet) { + const Index panels = numext::div_ceil(kc, Index(PacketTraits::WorkspaceRows)); + panelWidth = (numext::mini)(blockARows, rows * numext::div_ceil(numext::div_ceil(kc, panels), rows)); + colStep = colStep % packet == 0 ? colStep : packet % colStep == 0 ? packet : colStep * packet; + subcols = Index(l2 / (2 * std::ptrdiff_t(sizeof(Scalar)) * kc)); + } + } + subcols = numext::maxi<Index>(numext::round_down(subcols, colStep), colStep); for (Index j0 = 0; j0 < otherSize; j0 += nc) { const Index cols = (numext::mini)(otherSize - j0, nc); @@ -562,8 +615,8 @@ for (Index j2 = 0; j2 < cols; j2 += subcols) { Index actual_cols = (numext::mini)(cols - j2, subcols); // for each small vertical panels [T1k^T, T2k^T]^T of lhs - for (Index k1 = 0; k1 < actual_kc; k1 += SmallPanelWidth) { - Index actualPanelWidth = numext::mini<Index>(actual_kc - k1, SmallPanelWidth); + for (Index k1 = 0; k1 < actual_kc; k1 += panelWidth) { + Index actualPanelWidth = numext::mini<Index>(actual_kc - k1, panelWidth); // tr solve { Index i = IsLower ? k2 + k1 : k2 - k1 - 1; @@ -573,10 +626,17 @@ i = IsLower ? k2 + k1 : k2 - k1 - actualPanelWidth; } #endif - trsmKernelL<Scalar, Index, Mode, Conjugate, TriStorageOrder, OtherInnerStride, - /*Specialized=*/true>::kernel(actualPanelWidth, actual_cols, _tri + i + (i)*triStride, - triStride, _panel + i * otherIncr + j2 * otherStride, otherIncr, - otherStride); + using DiagonalKernel = + trsmKernelL<Scalar, Index, Mode, Conjugate, TriStorageOrder, OtherInnerStride, /*Specialized=*/true>; + const Scalar* diagonal = _tri + i + i * triStride; + Scalar* rhs = _panel + i * otherIncr + j2 * otherStride; + // Narrow panels keep their compile-time width bound, which lets compilers shorten the scalar + // kernel's loops; the packet kernel solves wider ones. + if (panelWidth == Index(SmallPanelWidth)) + DiagonalKernel::kernel(numext::mini<Index>(actualPanelWidth, SmallPanelWidth), actual_cols, diagonal, + triStride, rhs, otherIncr, otherStride); + else + DiagonalKernel::kernel(actualPanelWidth, actual_cols, diagonal, triStride, rhs, otherIncr, otherStride); } Index lengthTarget = actual_kc - k1 - actualPanelWidth;
diff --git a/Eigen/src/Core/util/ConfigureVectorization.h b/Eigen/src/Core/util/ConfigureVectorization.h index df2d236..51ce092 100644 --- a/Eigen/src/Core/util/ConfigureVectorization.h +++ b/Eigen/src/Core/util/ConfigureVectorization.h
@@ -432,6 +432,11 @@ #define EIGEN_VECTORIZE #define EIGEN_VECTORIZE_VSX 1 +// The POWER8 (ISA 2.07) vector instructions, such as 64-bit integer lane arithmetic and compares and vec_neg; a +// POWER7 target has VSX without them. +#if defined(__POWER8_VECTOR__) +#define EIGEN_VECTORIZE_POWER8_VECTOR +#endif #define EIGEN_VECTORIZE_FMA #include <altivec.h> // We need to #undef macros defined by <altivec.h> that conflict with standard C++ names. @@ -699,6 +704,8 @@ return "SSE, SSE2"; #elif defined(EIGEN_VECTORIZE_ALTIVEC) return "AltiVec"; +#elif defined(EIGEN_VECTORIZE_POWER8_VECTOR) + return "VSX, POWER8 vector"; #elif defined(EIGEN_VECTORIZE_VSX) return "VSX"; #elif defined(EIGEN_VECTORIZE_NEON)
diff --git a/Eigen/src/Core/util/Constants.h b/Eigen/src/Core/util/Constants.h index ea55eef..be1ebe8 100644 --- a/Eigen/src/Core/util/Constants.h +++ b/Eigen/src/Core/util/Constants.h
@@ -431,7 +431,10 @@ /** Use a QR decomposition with full pivoting as the first step. */ FullPivHouseholderQRPreconditioner = 0xC0, /** Used to disable the QR Preconditioner in BDCSVD. */ - DisableQRDecomposition = NoQRPreconditioner + DisableQRDecomposition = NoQRPreconditioner, + /** Used in the Options template parameter of JacobiSVD to apply the QR preconditioner to square matrices. + * BDCSVD forwards this option to the JacobiSVD it switches to for small inputs. */ + PreconditionSquareMatrix = 0x800 }; // JacobiSVD and BDCSVD combine QR preconditioner flags with decomposition flags in a single template bitmask.
diff --git a/Eigen/src/Core/util/GpuRuntime.h b/Eigen/src/Core/util/GpuRuntime.h index fb385dc..e9be11e 100644 --- a/Eigen/src/Core/util/GpuRuntime.h +++ b/Eigen/src/Core/util/GpuRuntime.h
@@ -41,21 +41,15 @@ } // namespace Eigen // EIGEN_GPU_RUNTIME_CHECK(call) evaluates a runtime-API call once and reports a failure through -// gpu_runtime_check_failed(). Define the macro before including the module to route failures elsewhere; define -// EIGEN_GPU_NO_RUNTIME_CHECKS to evaluate the call and ignore its result, which is what release builds did before. +// gpu_runtime_check_failed(). Define the macro before including the module to route failures elsewhere. There is +// no mode that ignores the result: a failed call has not done its work, and a sticky error leaves the context +// unusable for every later call. #if !defined(EIGEN_GPU_RUNTIME_CHECK) -#if defined(EIGEN_GPU_NO_RUNTIME_CHECKS) -#define EIGEN_GPU_RUNTIME_CHECK(expr) \ - do { \ - (void)(expr); \ - } while (0) -#else #define EIGEN_GPU_RUNTIME_CHECK(expr) \ do { \ ::Eigen::internal::gpu_runtime_check((expr), #expr, __FILE__, __LINE__); \ } while (0) #endif -#endif #if defined(EIGEN_GPUCC) namespace Eigen {
diff --git a/Eigen/src/Core/util/MKL_support.h b/Eigen/src/Core/util/MKL_support.h index 6ed22e3..74c02f1 100644 --- a/Eigen/src/Core/util/MKL_support.h +++ b/Eigen/src/Core/util/MKL_support.h
@@ -126,11 +126,13 @@ * EIGEN_64BIT_BLAS to select the 64-bit-integer ("ILP64") interface. It is an ABI-affecting macro and must be * consistent across all translation units. */ +#ifndef EIGEN_BLAS_INT #if defined(EIGEN_64BIT_BLAS) #define EIGEN_BLAS_INT std::int64_t #else #define EIGEN_BLAS_INT int #endif +#endif /* Suffix appended to external BLAS symbol names, for libraries that expose the 64-bit interface under decorated names * rather than in a separate library: reference LAPACK built with BUILD_INDEX64_EXT_API and oneMKL >= 2025 use _64
diff --git a/Eigen/src/Core/util/Macros.h b/Eigen/src/Core/util/Macros.h index 3f04bff..7733d72 100644 --- a/Eigen/src/Core/util/Macros.h +++ b/Eigen/src/Core/util/Macros.h
@@ -1205,6 +1205,20 @@ // "w" constraint will not compile. #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__("" : "+g"(X)); #endif +#elif EIGEN_ARCH_i386_OR_x86_64 && EIGEN_COMP_NVHPC +// nvc++ rejects a long double asm operand under every constraint, so its bytes go through memory instead. +namespace Eigen { +namespace internal { +template <typename T> +EIGEN_ALWAYS_INLINE void nvhpc_optimization_barrier(T& x) { + __asm__("" : "+g,x"(x)); +} +EIGEN_ALWAYS_INLINE void nvhpc_optimization_barrier(long double& x) { + __asm__("" : "+m"(*reinterpret_cast<unsigned char(*)[sizeof(long double)]>(&x))); +} +} // namespace internal +} // namespace Eigen +#define EIGEN_OPTIMIZATION_BARRIER(X) Eigen::internal::nvhpc_optimization_barrier(X); #elif EIGEN_ARCH_i386_OR_x86_64 // General, SSE. #define EIGEN_OPTIMIZATION_BARRIER(X) __asm__("" : "+g,x"(X));
diff --git a/Eigen/src/Core/util/Memory.h b/Eigen/src/Core/util/Memory.h index 15ca395..a29231d 100644 --- a/Eigen/src/Core/util/Memory.h +++ b/Eigen/src/Core/util/Memory.h
@@ -660,6 +660,54 @@ #undef EIGEN_ALLOCA #endif +// The SME GEMM kernel reads its packed panels a streaming vector at a time and slows down when they straddle +// 64-byte lines, so SME builds align the internal temporaries to 64 bytes, heap and stack alike. +#ifndef EIGEN_STACK_ALIGN_BYTES +#if defined(EIGEN_VECTORIZE_SME) && EIGEN_DEFAULT_ALIGN_BYTES < 64 +#define EIGEN_STACK_ALIGN_BYTES 64 +#else +#define EIGEN_STACK_ALIGN_BYTES EIGEN_DEFAULT_ALIGN_BYTES +#endif +#endif + +/** \internal Allocates a temporary buffer aligned to EIGEN_STACK_ALIGN_BYTES; release with scratch_free. */ +EIGEN_DEVICE_FUNC inline void* scratch_malloc(std::size_t size) { +#if EIGEN_STACK_ALIGN_BYTES > EIGEN_DEFAULT_ALIGN_BYTES + void* result = handmade_aligned_malloc(size, EIGEN_STACK_ALIGN_BYTES); + if (!result && size) throw_std_bad_alloc(); + return result; +#else + return aligned_malloc(size); +#endif +} +EIGEN_DEVICE_FUNC inline void scratch_free(void* ptr) { +#if EIGEN_STACK_ALIGN_BYTES > EIGEN_DEFAULT_ALIGN_BYTES + handmade_aligned_free(ptr); +#else + aligned_free(ptr); +#endif +} + +/** \internal Like aligned_new, at the temporaries' alignment (EIGEN_STACK_ALIGN_BYTES); release with scratch_delete. */ +template <typename T> +EIGEN_DEVICE_FUNC inline T* scratch_new(std::size_t size) { + check_size_for_overflow<T>(size); + T* result = static_cast<T*>(scratch_malloc(sizeof(T) * size)); + EIGEN_TRY { default_construct_elements_of_array(result, size); } + EIGEN_CATCH(...) { + scratch_free(result); + EIGEN_THROW; + } + return result; +} + +/** \internal Deletes objects constructed with scratch_new. */ +template <typename T> +EIGEN_DEVICE_FUNC inline void scratch_delete(T* ptr, std::size_t size) { + destruct_elements_of_array<T>(ptr, size); + scratch_free(ptr); +} + // This helper class construct the allocated memory, and takes care of destructing and freeing the handled data // at destruction time. In practice this helper class is mainly useful to avoid memory leak in case of exceptions. template <typename T> @@ -684,7 +732,7 @@ EIGEN_IF_CONSTEXPR (NumTraits<T>::RequireInitialization) { if (m_ptr) Eigen::internal::destruct_elements_of_array<T>(m_ptr, m_size); } - if (m_deallocate) Eigen::internal::aligned_free(m_ptr); + if (m_deallocate) Eigen::internal::scratch_free(m_ptr); } protected: @@ -768,18 +816,6 @@ */ #if defined(EIGEN_ALLOCA) && !defined(EIGEN_NO_ALLOCA) -// Stack temporaries hold the GEMM packed panels. The SME kernel loads them a -// streaming vector at a time and runs 35-50% slower when they straddle -// 64-byte lines, so SME builds align them to that rather than to the ABI's -// EIGEN_DEFAULT_ALIGN_BYTES. -#ifndef EIGEN_STACK_ALIGN_BYTES -#if defined(EIGEN_VECTORIZE_SME) && EIGEN_DEFAULT_ALIGN_BYTES < 64 -#define EIGEN_STACK_ALIGN_BYTES 64 -#else -#define EIGEN_STACK_ALIGN_BYTES EIGEN_DEFAULT_ALIGN_BYTES -#endif -#endif - #if EIGEN_STACK_ALIGN_BYTES > 0 // We always manually re-align the result of EIGEN_ALLOCA. // If alloca is already aligned, the compiler should be smart enough to optimize away the re-alignment. @@ -806,7 +842,7 @@ TYPE* NAME = (BUFFER) != 0 ? (BUFFER) \ : reinterpret_cast<TYPE*>((sizeof(TYPE) * (SIZE) <= EIGEN_STACK_ALLOCATION_LIMIT) \ ? EIGEN_ALIGNED_ALLOCA(sizeof(TYPE) * (SIZE)) \ - : Eigen::internal::aligned_malloc(sizeof(TYPE) * (SIZE))); \ + : Eigen::internal::scratch_malloc(sizeof(TYPE) * (SIZE))); \ Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \ (BUFFER) == 0 ? NAME : 0, SIZE, sizeof(TYPE) * (SIZE) > EIGEN_STACK_ALLOCATION_LIMIT) @@ -824,7 +860,7 @@ #define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \ Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \ TYPE* NAME = \ - (BUFFER) != 0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::aligned_malloc(sizeof(TYPE) * (SIZE))); \ + (BUFFER) != 0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::scratch_malloc(sizeof(TYPE) * (SIZE))); \ Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \ (BUFFER) == 0 ? NAME : 0, SIZE, true)
diff --git a/Eigen/src/Eigenvalues/RealSchur.h b/Eigen/src/Eigenvalues/RealSchur.h index 0493dde..2349c9e 100644 --- a/Eigen/src/Eigenvalues/RealSchur.h +++ b/Eigen/src/Eigenvalues/RealSchur.h
@@ -590,30 +590,71 @@ const Index size = matT.cols(); - for (Index k = im; k <= iu - 2; ++k) { - bool firstIteration = (k == im); + // Reflectors are chased in windows k0 <= k < k1 of at most WindowSize. Reflector k reads column k - 1 and its + // right-hand update reaches column k + 2, so only columns < k1 + 2 feed later reflectors of the window. The left + // updates of the columns beyond are deferred to the window's end and applied column by column in the original + // order: the same arithmetic, on contiguous segments instead of one strided three-row pass per reflector. + constexpr Index WindowSize = 32; + constexpr bool deferLeft = !TMatrix::IsRowMajor && int(TMatrix::InnerStrideAtCompileTime) == 1; + Index windowK[WindowSize]; + Scalar windowTau[WindowSize]; + Matrix<Scalar, 2, 1> windowEss[WindowSize]; - Vector3s v; - if (firstIteration) - v = firstHouseholderVector; - else - v = matT.template block<3, 1>(k, k - 1); + for (Index k0 = im; k0 <= iu - 2; k0 += WindowSize) { + const Index k1 = (std::min)(k0 + WindowSize, iu - 1); + const Index nearEnd = deferLeft ? (std::min)(size, k1 + 2) : size; + Index numDeferred = 0; + for (Index k = k0; k < k1; ++k) { + bool firstIteration = (k == im); - Scalar tau, beta; - Matrix<Scalar, 2, 1> ess; - v.makeHouseholder(ess, tau, beta); + Vector3s v; + if (firstIteration) + v = firstHouseholderVector; + else + v = matT.template block<3, 1>(k, k - 1); - if (!numext::is_exactly_zero(beta)) // if v is not zero - { - if (firstIteration && k > il) - matT.coeffRef(k, k - 1) = -matT.coeff(k, k - 1); - else if (!firstIteration) - matT.coeffRef(k, k - 1) = beta; + Scalar tau, beta; + Matrix<Scalar, 2, 1> ess; + v.makeHouseholder(ess, tau, beta); - // These Householder transformations form the O(n^3) part of the algorithm - matT.block(k, k, 3, size - k).applyHouseholderOnTheLeft(ess, tau, workspace); - matT.block(0, k, (std::min)(iu, k + 3) + 1, 3).applyHouseholderOnTheRight(ess, tau, workspace); - if (computeU) m_matU.block(0, k, size, 3).applyHouseholderOnTheRight(ess, tau, workspace); + if (!numext::is_exactly_zero(beta)) // if v is not zero + { + if (firstIteration && k > il) + matT.coeffRef(k, k - 1) = -matT.coeff(k, k - 1); + else if (!firstIteration) + matT.coeffRef(k, k - 1) = beta; + + // These Householder transformations form the O(n^3) part of the algorithm + matT.block(k, k, 3, nearEnd - k).applyHouseholderOnTheLeft(ess, tau, workspace); + matT.block(0, k, (std::min)(iu, k + 3) + 1, 3).applyHouseholderOnTheRight(ess, tau, workspace); + if (computeU) m_matU.block(0, k, size, 3).applyHouseholderOnTheRight(ess, tau, workspace); + // A zero tau is the identity, which the immediate path skips; applying it would turn -0 into +0 and Inf + // into NaN. + if (nearEnd < size && !numext::is_exactly_zero(tau)) { + windowK[numDeferred] = k; + windowTau[numDeferred] = tau; + windowEss[numDeferred] = ess; + ++numDeferred; + } + } + } + // Groups of columns give independent dependency chains: within one column, each update reads two entries the + // previous one wrote. + constexpr Index ColumnGroup = 16; + for (Index j0 = nearEnd; j0 < size && numDeferred > 0; j0 += ColumnGroup) { + const Index j1 = (std::min)(j0 + ColumnGroup, size); + for (Index r = 0; r < numDeferred; ++r) { + const Index k = windowK[r]; + const Scalar e0 = windowEss[r].coeff(0), e1 = windowEss[r].coeff(1), tau = windowTau[r]; + for (Index j = j0; j < j1; ++j) { + // Column-major with unit inner stride: rows k, k + 1, k + 2 of column j are consecutive. + Scalar* x = &matT.coeffRef(k, j); + const Scalar tmp = tau * ((e0 * x[1] + e1 * x[2]) + x[0]); + x[0] -= tmp; + x[1] -= e0 * tmp; + x[2] -= e1 * tmp; + } + } } }
diff --git a/Eigen/src/Eigenvalues/TridiagonalBisection.h b/Eigen/src/Eigenvalues/TridiagonalBisection.h index e600c86..87e1cfc 100644 --- a/Eigen/src/Eigenvalues/TridiagonalBisection.h +++ b/Eigen/src/Eigenvalues/TridiagonalBisection.h
@@ -377,13 +377,22 @@ // off-diagonal and during the Sturm recurrence; eigenvalues scale linearly, so the // scaling is undone at the very end. (The caller has already verified the input is // finite.) This mirrors the uniform scaling done in SelfAdjointEigenSolver::compute(). - RealScalar maxCoeff = diag.cwiseAbs().maxCoeff(); - if (n >= 2) maxCoeff = numext::maxi(maxCoeff, subdiag.cwiseAbs().maxCoeff()); + // The rescan recovers an all-subnormal input that a flushing SIMD unit reads as zero. + RealScalar maxCoeff = safe_scaling<RealScalar>::recover_flushed_max_coeff(diag, diag.cwiseAbs().maxCoeff()); + if (n >= 2) { + maxCoeff = max_preserving_subnormals( + maxCoeff, safe_scaling<RealScalar>::recover_flushed_max_coeff(subdiag, subdiag.cwiseAbs().maxCoeff())); + } // Local contiguous copies of the scaled matrix data, |off-diagonal|, and its square. ArrayType alpha(n), beta_abs(n - 1); const auto factors = safe_scaling<RealScalar>::scale_to(alpha, diag.array(), maxCoeff); - if (n >= 2) safe_scaling<RealScalar>::scale_to(beta_abs, subdiag.array().abs(), maxCoeff, factors); + if (n >= 2) { + safe_scaling<RealScalar>::scale_to(beta_abs, subdiag.array(), maxCoeff, factors); + // |e| from the representation after scaling: MSVC's fabsf goes through double, and narrowing a subnormal result + // back to float flushes it to zero under FTZ. + for (Index i = 0; i < n - 1; ++i) beta_abs(i) = abs_preserving_subnormals(beta_abs(i)); + } const RealScalar scale = factors.scale; const ArrayType beta_sq = (n >= 2) ? ArrayType(beta_abs.square()) : ArrayType(0);
diff --git a/Eigen/src/Eigenvalues/TridiagonalInverseIteration.h b/Eigen/src/Eigenvalues/TridiagonalInverseIteration.h index a117d0f..12d0349 100644 --- a/Eigen/src/Eigenvalues/TridiagonalInverseIteration.h +++ b/Eigen/src/Eigenvalues/TridiagonalInverseIteration.h
@@ -361,7 +361,10 @@ // Normalize T (and the shifts) to O(1) so the deliberately near-singular factor/solve cannot // overflow or underflow; eigenvectors are invariant under this uniform scaling. - const RealScalar maxCoeff = numext::maxi(diag.cwiseAbs().maxCoeff(), subdiag.cwiseAbs().maxCoeff()); + // The rescan recovers an all-subnormal input that a flushing SIMD unit reads as zero. + const RealScalar maxCoeff = max_preserving_subnormals( + safe_scaling<RealScalar>::recover_flushed_max_coeff(diag, diag.cwiseAbs().maxCoeff()), + safe_scaling<RealScalar>::recover_flushed_max_coeff(subdiag, subdiag.cwiseAbs().maxCoeff())); Matrix<RealScalar, Dynamic, 1> sdiag(n), ssub(n - 1); const auto factors = safe_scaling<RealScalar>::scale_to(sdiag, diag, maxCoeff); safe_scaling<RealScalar>::scale_to(ssub, subdiag, maxCoeff, factors); @@ -516,6 +519,30 @@ const Index m = eivals.size(); if (n == 0 || m == 0) return 0; + // A matrix whose largest entry is below the recovery threshold min / eps can hold significant subnormal + // couplings, which FTZ/DAZ hardware reads as zero in the comparisons below (ARMv7 NEON also in the packet maxima). + // Scale it into the normal range first, exactly, through integer significands: the eigenvectors are invariant + // under the scaling, and every representable entry becomes normal. The test reads the exponent from the + // representation, since the maximum itself may be subnormal and a floating-point comparison on it is what FTZ/DAZ + // breaks; a zero or non-finite maximum has no exponent in range and is left alone. + { + RealScalar maxCoeff = safe_scaling<RealScalar>::recover_flushed_max_coeff(diag, diag.cwiseAbs().maxCoeff()); + if (n >= 2) { + maxCoeff = max_preserving_subnormals( + maxCoeff, safe_scaling<RealScalar>::recover_flushed_max_coeff(subdiag, subdiag.cwiseAbs().maxCoeff())); + } + // 2^(e - 1) <= maxCoeff < 2^e, so maxCoeff < 2^recovery iff e <= recovery; e == 0 for a zero maximum and one + // above the largest finite exponent for infinities and NaN. + const int e = frexp_exponent_preserving_subnormals(maxCoeff); + if (e != 0 && e <= safe_scaling<RealScalar>::subnormal_recovery_exponent()) { + VectorType sdiag(n), ssub(n - 1), seivals(m); + const auto factors = safe_scaling<RealScalar>::scale_to(sdiag, diag, maxCoeff); + safe_scaling<RealScalar>::scale_to(ssub, subdiag, maxCoeff, factors); + safe_scaling<RealScalar>::scale_to(seivals, eivals, maxCoeff, factors); + return tridiagonal_inverse_iteration(sdiag, ssub, seivals, eivecs); + } + } + // Split at negligible couplings (cf. xSTEBZ): |e_k| <= eps * sqrt(|d_k|) * sqrt(|d_k+1|). The // geometric-mean form is scale-invariant on its own (both sides scale linearly) and needs no // additive floor: a floor expressed at any single scale falsely splits strongly connected blocks
diff --git a/Eigen/src/Geometry/AngleAxis.h b/Eigen/src/Geometry/AngleAxis.h index 2796486..c190c52 100644 --- a/Eigen/src/Geometry/AngleAxis.h +++ b/Eigen/src/Geometry/AngleAxis.h
@@ -213,6 +213,7 @@ template <typename Scalar> template <typename Derived> EIGEN_DEVICE_FUNC AngleAxis<Scalar>& AngleAxis<Scalar>::operator=(const MatrixBase<Derived>& mat) { + // A 4D vector holds quaternion coefficients; anything else is a 3x3 rotation matrix. if (mat.size() == 4) { return *this = QuaternionType(mat); } @@ -221,69 +222,24 @@ /** * \brief Sets \c *this from a 3x3 rotation matrix. + * + * The resulting axis is normalized, and the computed angle is in the [0,pi] range. **/ template <typename Scalar> template <typename Derived> EIGEN_DEVICE_FUNC AngleAxis<Scalar>& AngleAxis<Scalar>::fromRotationMatrix(const MatrixBase<Derived>& mat) { - EIGEN_USING_STD(atan2) - EIGEN_USING_STD(sqrt) EIGEN_STATIC_ASSERT( (std::is_same<Scalar, typename Derived::Scalar>::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) eigen_assert(mat.cols() == 3 && mat.rows() == 3); - const typename internal::nested_eval<Derived, 3>::type m(mat.derived()); - - // Skew-symmetric part gives sin(angle) * axis. - const Scalar sx = m.coeff(2, 1) - m.coeff(1, 2); - const Scalar sy = m.coeff(0, 2) - m.coeff(2, 0); - const Scalar sz = m.coeff(1, 0) - m.coeff(0, 1); - const Scalar s = sqrt(sx * sx + sy * sy + sz * sz); // = 2*sin(angle) - - // trace = 1 + 2*cos(angle) - const Scalar c = m.trace() - Scalar(1); // = 2*cos(angle) - - // Use atan2 for the angle: accurate at all angles including near 0 and pi. - m_angle = atan2(s, c); - - // Use the skew-symmetric part only when sin(angle) is large enough for - // accurate axis extraction. Near angle=0 or angle=pi, sin(angle) is small - // and the axis must be computed differently. - const Scalar sin_threshold = sqrt(NumTraits<Scalar>::epsilon()); - if (s > sin_threshold) { - // General case: axis from skew-symmetric part. - const Scalar inv_s = Scalar(1) / s; - m_axis << sx * inv_s, sy * inv_s, sz * inv_s; - } else if (c > Scalar(0)) { - // Near identity (angle ≈ 0): axis is arbitrary, use (1,0,0). - m_axis << Scalar(1), Scalar(0), Scalar(0); - } else { - // Near angle = pi: extract axis from the symmetric part (R + I) / 2. - // The axis is the eigenvector corresponding to eigenvalue 1. - // Use the column of (R + I) with the largest diagonal entry for robustness. - const Scalar d0 = m.coeff(0, 0); - const Scalar d1 = m.coeff(1, 1); - const Scalar d2 = m.coeff(2, 2); - if (d0 >= d1 && d0 >= d2) { - // x is the largest component - const Scalar x = sqrt(numext::maxi(d0 - d1 - d2 + Scalar(1), Scalar(0)) * Scalar(0.5)); - const Scalar inv_2x = Scalar(0.5) / (x + NumTraits<Scalar>::epsilon()); - m_axis << x, (m.coeff(0, 1) + m.coeff(1, 0)) * inv_2x, (m.coeff(0, 2) + m.coeff(2, 0)) * inv_2x; - } else if (d1 >= d2) { - // y is the largest component - const Scalar y = sqrt(numext::maxi(d1 - d0 - d2 + Scalar(1), Scalar(0)) * Scalar(0.5)); - const Scalar inv_2y = Scalar(0.5) / (y + NumTraits<Scalar>::epsilon()); - m_axis << (m.coeff(0, 1) + m.coeff(1, 0)) * inv_2y, y, (m.coeff(1, 2) + m.coeff(2, 1)) * inv_2y; - } else { - // z is the largest component - const Scalar z = sqrt(numext::maxi(d2 - d0 - d1 + Scalar(1), Scalar(0)) * Scalar(0.5)); - const Scalar inv_2z = Scalar(0.5) / (z + NumTraits<Scalar>::epsilon()); - m_axis << (m.coeff(0, 2) + m.coeff(2, 0)) * inv_2z, (m.coeff(1, 2) + m.coeff(2, 1)) * inv_2z, z; - } - m_axis.normalize(); - } - - return *this; + // Compute a (non-unit) quaternion proportional to the rotation using + // Shoemake's algorithm (shared with Quaternion::operator=(MatrixBase)), + // skipping the intermediate normalization since AngleAxis::operator=(QuaternionBase) + // normalizes the axis and uses atan2(|v|, |w|). + QuaternionType q; + internal::quaternionbase_assign_impl<Derived, 3, 3>::template run<false>(q, mat.derived()); + return *this = q; } /** Constructs and \returns an equivalent 3x3 rotation matrix.
diff --git a/Eigen/src/Geometry/Quaternion.h b/Eigen/src/Geometry/Quaternion.h index e0579c5..f2c6dbc 100644 --- a/Eigen/src/Geometry/Quaternion.h +++ b/Eigen/src/Geometry/Quaternion.h
@@ -968,34 +968,50 @@ template <typename Other> struct quaternionbase_assign_impl<Other, 3, 3> { using Scalar = typename Other::Scalar; - template <class Derived> + + template <bool Normalize> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar compute_scale(Scalar& t) { + if (!Normalize) return Scalar(1); + t = numext::sqrt(t); + Scalar s = Scalar(0.5) / t; + t *= Scalar(0.5); + return s; + } + + template <int AxisI, int AxisJ, int AxisK, bool Normalize, class Derived, class Mat> + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void assign_branch(QuaternionBase<Derived>& q, const Mat& mat, + Scalar mii, Scalar mjj, Scalar mkk) { + // Guard against slightly negative argument from non-orthogonal matrices. + Scalar t = numext::maxi(mii - mjj - mkk + Scalar(1.0), Scalar(0)); + const Scalar s = compute_scale<Normalize>(t); + q.coeffs().coeffRef(AxisI) = t; + q.w() = (mat.coeff(AxisK, AxisJ) - mat.coeff(AxisJ, AxisK)) * s; + q.coeffs().coeffRef(AxisJ) = (mat.coeff(AxisJ, AxisI) + mat.coeff(AxisI, AxisJ)) * s; + q.coeffs().coeffRef(AxisK) = (mat.coeff(AxisK, AxisI) + mat.coeff(AxisI, AxisK)) * s; + } + + template <bool Normalize = true, class Derived> EIGEN_DEVICE_FUNC static inline void run(QuaternionBase<Derived>& q, const Other& a_mat) { const typename internal::nested_eval<Other, 2>::type mat(a_mat); - EIGEN_USING_STD(sqrt) // This algorithm comes from "Quaternion Calculus and Fast Animation", // Ken Shoemake, 1987 SIGGRAPH course notes - Scalar t = mat.trace(); + const Scalar m00 = mat.coeff(0, 0); + const Scalar m11 = mat.coeff(1, 1); + const Scalar m22 = mat.coeff(2, 2); + Scalar t = m00 + (m11 + m22); if (t > Scalar(0)) { - t = sqrt(numext::maxi(t + Scalar(1.0), Scalar(0))); - q.w() = Scalar(0.5) * t; - t = Scalar(0.5) / t; - q.x() = (mat.coeff(2, 1) - mat.coeff(1, 2)) * t; - q.y() = (mat.coeff(0, 2) - mat.coeff(2, 0)) * t; - q.z() = (mat.coeff(1, 0) - mat.coeff(0, 1)) * t; + t = numext::maxi(t + Scalar(1.0), Scalar(0)); + const Scalar s = compute_scale<Normalize>(t); + q.w() = t; + q.x() = (mat.coeff(2, 1) - mat.coeff(1, 2)) * s; + q.y() = (mat.coeff(0, 2) - mat.coeff(2, 0)) * s; + q.z() = (mat.coeff(1, 0) - mat.coeff(0, 1)) * s; + } else if (m00 >= m11 && m00 >= m22) { + assign_branch<0, 1, 2, Normalize>(q, mat, m00, m11, m22); + } else if (m11 >= m22) { + assign_branch<1, 2, 0, Normalize>(q, mat, m11, m22, m00); } else { - Index i = 0; - if (mat.coeff(1, 1) > mat.coeff(0, 0)) i = 1; - if (mat.coeff(2, 2) > mat.coeff(i, i)) i = 2; - Index j = (i + 1) % 3; - Index k = (j + 1) % 3; - - // Guard against slightly negative argument from non-orthogonal matrices. - t = sqrt(numext::maxi(mat.coeff(i, i) - mat.coeff(j, j) - mat.coeff(k, k) + Scalar(1.0), Scalar(0))); - q.coeffs().coeffRef(i) = Scalar(0.5) * t; - t = Scalar(0.5) / t; - q.w() = (mat.coeff(k, j) - mat.coeff(j, k)) * t; - q.coeffs().coeffRef(j) = (mat.coeff(j, i) + mat.coeff(i, j)) * t; - q.coeffs().coeffRef(k) = (mat.coeff(k, i) + mat.coeff(i, k)) * t; + assign_branch<2, 0, 1, Normalize>(q, mat, m22, m00, m11); } } };
diff --git a/Eigen/src/Householder/BlockHouseholder.h b/Eigen/src/Householder/BlockHouseholder.h index 9e99c85..0614e72 100644 --- a/Eigen/src/Householder/BlockHouseholder.h +++ b/Eigen/src/Householder/BlockHouseholder.h
@@ -117,7 +117,7 @@ // tmp = mat * V, split along V's top/bottom partition (see the left-apply for context). Matrix<typename MatrixType::Scalar, MatrixType::RowsAtCompileTime, VectorsType::ColsAtCompileTime, - (MatrixType::MaxRowsAtCompileTime == 1 && VectorsType::MaxColsAtCompileTime != 1) ? ColMajor : RowMajor, + (VectorsType::MaxColsAtCompileTime == 1 && MatrixType::MaxRowsAtCompileTime != 1) ? ColMajor : RowMajor, MatrixType::MaxRowsAtCompileTime, VectorsType::MaxColsAtCompileTime> tmp(mat.rows(), nbVecs); tmp.noalias() = mat.leftCols(nbVecs) * V_top.template triangularView<UnitLower>();
diff --git a/Eigen/src/Householder/Householder.h b/Eigen/src/Householder/Householder.h index f977a29..6e98552 100644 --- a/Eigen/src/Householder/Householder.h +++ b/Eigen/src/Householder/Householder.h
@@ -165,7 +165,7 @@ // not survive -ffinite-math-only, which folds that test away, whereas a comparison against a finite bound is // still evaluated. const RealScalar sqNormBound = NumTraits<RealScalar>::highest() / RealScalar(2); - const RealScalar componentBound = numext::sqrt(sqNormBound / RealScalar(2)); + const RealScalar componentBound = numext::sqrt<RealScalar>(sqNormBound / RealScalar(2)); const RealScalar c0Max = numext::maxi(numext::abs(numext::real(c0)), numext::abs(numext::imag(c0))); unscaledSqNormOverflows = !(c0Max <= componentBound) || !(tailSqNorm <= sqNormBound); } @@ -230,7 +230,7 @@ tau = conj(Scalar(RealScalar(1)) - scaledC0 / scaledBeta); return; } - beta = numext::sqrt(numext::abs2(c0) + tailSqNorm); + beta = numext::sqrt<RealScalar>(numext::abs2(c0) + tailSqNorm); if (numext::real(c0) >= RealScalar(0)) beta = -beta; essential = tail.unwind() / (c0 - beta); tau = conj((beta - c0) / beta); @@ -275,6 +275,75 @@ } }; +template <typename Derived, typename EssentialPart, + bool Fused = !Derived::IsRowMajor && bool(traits<Derived>::Flags & DirectAccessBit) && + inner_stride_at_compile_time<Derived>::value == 1 && + packet_traits<typename Derived::Scalar>::Vectorizable && EssentialPart::ColsAtCompileTime == 1 && + (EssentialPart::RowsAtCompileTime == 1 || EssentialPart::RowsAtCompileTime == 2)> +struct householder_apply_right_impl { + using Scalar = typename Derived::Scalar; + static EIGEN_DEVICE_FUNC void run(MatrixBase<Derived>& mat, const EssentialPart& essential, const Scalar& tau, + Scalar* workspace) { + Map<typename plain_col_type<typename Derived::PlainObject>::type> tmp(workspace, mat.rows()); + Block<Derived, Derived::RowsAtCompileTime, EssentialPart::SizeAtCompileTime> right(mat.derived(), 0, 1, mat.rows(), + mat.cols() - 1); + tmp.noalias() = right.unwind() * essential; + // M H = M - (M v) tau v^*: tau multiplies from the right, as in the one-column branch. + tmp = (tmp + mat.col(0)) * tau; + mat.col(0) = mat.col(0) - tmp; + right.unwind().noalias() -= tmp * essential.adjoint(); + } +}; + +// Two- and three-element reflectors on contiguous columns: one pass down the rows updates all the columns, where the +// general path makes four passes through a temporary. These are the O(n^3) updates of the Francis QR step. +template <typename Derived, typename EssentialPart> +struct householder_apply_right_impl<Derived, EssentialPart, true> { + using Scalar = typename Derived::Scalar; + using Packet = typename packet_traits<Scalar>::type; + static constexpr int K = EssentialPart::RowsAtCompileTime; + + static EIGEN_DEVICE_FUNC void run(MatrixBase<Derived>& mat, const EssentialPart& essential, const Scalar& tau, + Scalar*) { + constexpr Index PacketSize = unpacket_traits<Packet>::size; + eigen_assert(mat.cols() == K + 1); + // Copied first: tau and the essential part may alias coefficients of mat. + Scalar v[K], vc[K]; + for (int j = 0; j < K; ++j) { + v[j] = essential.coeff(j); + vc[j] = numext::conj(v[j]); + } + const Scalar tauValue = tau; + const Index rows = mat.rows(); + Scalar* col[K + 1]; + for (int j = 0; j <= K; ++j) col[j] = &mat.derived().coeffRef(0, j); + + const Packet ptau = pset1<Packet>(tauValue); + Packet pv[K], pvc[K]; + for (int j = 0; j < K; ++j) { + pv[j] = pset1<Packet>(v[j]); + pvc[j] = pset1<Packet>(vc[j]); + } + const Index vectorEnd = numext::round_down(rows, PacketSize); + for (Index i = 0; i < vectorEnd; i += PacketSize) { + Packet x[K + 1]; + for (int j = 0; j <= K; ++j) x[j] = ploadu<Packet>(col[j] + i); + Packet t = x[0]; + for (int j = 0; j < K; ++j) t = pmadd(x[j + 1], pv[j], t); + t = pmul(t, ptau); + pstoreu(col[0] + i, psub(x[0], t)); + for (int j = 0; j < K; ++j) pstoreu(col[j + 1] + i, pnmadd(t, pvc[j], x[j + 1])); + } + for (Index i = vectorEnd; i < rows; ++i) { + Scalar t = col[0][i]; + for (int j = 0; j < K; ++j) t += col[j + 1][i] * v[j]; + t *= tauValue; + col[0][i] -= t; + for (int j = 0; j < K; ++j) col[j + 1][i] -= t * vc[j]; + } + } +}; + } // namespace internal /** Apply the elementary reflector H given by @@ -297,7 +366,8 @@ EIGEN_DEVICE_FUNC void MatrixBase<Derived>::applyHouseholderOnTheLeft(const EssentialPart& essential, const Scalar& tau, Scalar* workspace) { if (rows() == 1) { - *this *= Scalar(1) - tau; + // H = 1 - tau multiplies from the left, which a noncommutative scalar distinguishes. + *this = (Scalar(1) - tau) * *this; } else if (!numext::is_exactly_zero(tau)) { internal::householder_apply_left_impl<Derived, EssentialPart>::run(*this, essential, tau, workspace); } @@ -325,13 +395,7 @@ if (cols() == 1) { *this *= Scalar(1) - tau; } else if (!numext::is_exactly_zero(tau)) { - Map<typename internal::plain_col_type<PlainObject>::type> tmp(workspace, rows()); - Block<Derived, Derived::RowsAtCompileTime, EssentialPart::SizeAtCompileTime> right(derived(), 0, 1, rows(), - cols() - 1); - tmp.noalias() = right.unwind() * essential; - tmp = tau * (tmp + this->col(0)); - this->col(0) = this->col(0) - tmp; - right.unwind().noalias() -= tmp * essential.adjoint(); + internal::householder_apply_right_impl<Derived, EssentialPart>::run(*this, essential, tau, workspace); } }
diff --git a/Eigen/src/Householder/HouseholderSequence.h b/Eigen/src/Householder/HouseholderSequence.h index 4c3bb06..99f081c 100644 --- a/Eigen/src/Householder/HouseholderSequence.h +++ b/Eigen/src/Householder/HouseholderSequence.h
@@ -107,10 +107,33 @@ template <typename OtherScalarType, typename MatrixType> struct matrix_type_times_scalar_type { using ResultScalar = typename ScalarBinaryOpTraits<OtherScalarType, typename MatrixType::Scalar>::ReturnType; - using Type = Matrix<ResultScalar, MatrixType::RowsAtCompileTime, MatrixType::ColsAtCompileTime, 0, - MatrixType::MaxRowsAtCompileTime, MatrixType::MaxColsAtCompileTime>; + using Type = + Matrix<ResultScalar, MatrixType::RowsAtCompileTime, MatrixType::ColsAtCompileTime, + (MatrixType::MaxRowsAtCompileTime == 1 && MatrixType::MaxColsAtCompileTime != 1) ? RowMajor : ColMajor, + MatrixType::MaxRowsAtCompileTime, MatrixType::MaxColsAtCompileTime>; }; +// A permutation has no scalar type; its product with a Householder sequence takes the sequence's. +template <typename Scalar, typename PermutationType> +struct permutation_type_as_matrix_type { + using Type = Matrix<Scalar, PermutationType::RowsAtCompileTime, PermutationType::ColsAtCompileTime, 0, + PermutationType::MaxRowsAtCompileTime, PermutationType::MaxColsAtCompileTime>; +}; + +// A diagonal or triangular operand is assigned to the result directly when the scalar types agree. The triangular +// assignment kernel does not convert scalars, so a promoted result evaluates the operand and casts it, as the dense +// product casts its operand. +template <typename ResultType, typename OtherDerived> +std::enable_if_t<std::is_same<typename ResultType::Scalar, typename OtherDerived::Scalar>::value> +assign_householder_operand(ResultType& res, const EigenBase<OtherDerived>& other) { + res = other.derived(); +} +template <typename ResultType, typename OtherDerived> +std::enable_if_t<!std::is_same<typename ResultType::Scalar, typename OtherDerived::Scalar>::value> +assign_householder_operand(ResultType& res, const EigenBase<OtherDerived>& other) { + res = other.derived().toDenseMatrix().template cast<typename ResultType::Scalar>(); +} + } // end namespace internal template <typename VectorsType, typename CoeffsType, int Side> @@ -419,9 +442,78 @@ return res; } + /** \brief Computes the product of a Householder sequence with a diagonal matrix. + * + * The reflections are applied to the diagonal matrix on the path the dense product uses for an identity + * operand, so \f$ HD \f$ costs no more than \f$ H \f$ itself. + */ + template <typename OtherDerived> + typename internal::matrix_type_times_scalar_type<Scalar, OtherDerived>::Type operator*( + const DiagonalBase<OtherDerived>& other) const { + return applyToUpperTriangularOnTheLeft< + typename internal::matrix_type_times_scalar_type<Scalar, OtherDerived>::Type>(other, true); + } + + /** \brief Computes the product of a Householder sequence with a triangular or self-adjoint view. + * + * A square upper triangular operand takes the same path as a diagonal one; any other view is evaluated into a + * dense matrix and multiplied as such. + */ + template <typename OtherDerived> + typename internal::matrix_type_times_scalar_type<Scalar, OtherDerived>::Type operator*( + const TriangularBase<OtherDerived>& other) const { + constexpr bool kUpperTriangular = + (int(OtherDerived::Mode) & (int(Upper) | int(Lower) | int(SelfAdjoint))) == int(Upper); + return applyToUpperTriangularOnTheLeft< + typename internal::matrix_type_times_scalar_type<Scalar, OtherDerived>::Type>(other, kUpperTriangular); + } + + /** \brief Computes the product of a Householder sequence with a permutation matrix. + * + * \f$ HP \f$ is \f$ H \f$ with its columns permuted: the sequence is evaluated and the permutation applied in + * place. The result takes the sequence's scalar type. + */ + template <typename OtherDerived> + typename internal::permutation_type_as_matrix_type<Scalar, OtherDerived>::Type operator*( + const PermutationBase<OtherDerived>& other) const { + using ResultType = typename internal::permutation_type_as_matrix_type<Scalar, OtherDerived>::Type; + eigen_assert(cols() == other.rows()); + ResultType res = ResultType::Identity(rows(), rows()); + applyThisOnTheLeft(res, true); + // The permutation product runs in place when its operand is the destination. + res.noalias() = res * other.derived(); + return res; + } + + /** \brief Computes the product of a Householder sequence with the inverse of a permutation matrix. */ + template <typename OtherDerived> + typename internal::permutation_type_as_matrix_type<Scalar, Inverse<OtherDerived>>::Type operator*( + const InverseImpl<OtherDerived, PermutationStorage>& other) const { + using ResultType = typename internal::permutation_type_as_matrix_type<Scalar, Inverse<OtherDerived>>::Type; + eigen_assert(cols() == other.rows()); + ResultType res = ResultType::Identity(rows(), rows()); + applyThisOnTheLeft(res, true); + res.noalias() = res * other.derived(); + return res; + } + template <typename VectorsType_, typename CoeffsType_, int Side_> friend struct internal::hseq_side_dependent_impl; + private: + // The identity path of applyThisOnTheLeft() skips the columns left of the rows each reflection acts on, which is + // valid for any square input that vanishes below the diagonal: a diagonal or upper triangular matrix as much as + // the identity. Other inputs take the full path. + template <typename ResultType, typename OtherDerived> + ResultType applyToUpperTriangularOnTheLeft(const EigenBase<OtherDerived>& other, bool upperTriangular) const { + eigen_assert(cols() == other.rows()); + ResultType res; + internal::assign_householder_operand(res, other); + applyThisOnTheLeft(res, upperTriangular && res.rows() == res.cols()); + return res; + } + + public: /** \brief Sets the length of the Householder sequence. * \param [in] length New value for the length. * @@ -508,6 +600,70 @@ return res; } +/** \brief Computes the product of a diagonal matrix with a Householder sequence. + * + * \f$ DH \f$ is \f$ H \f$ with its rows scaled: the sequence is evaluated and scaled in place. + */ +template <typename OtherDerived, typename VectorsType, typename CoeffsType, int Side> +typename internal::matrix_type_times_scalar_type<typename VectorsType::Scalar, OtherDerived>::Type operator*( + const DiagonalBase<OtherDerived>& other, const HouseholderSequence<VectorsType, CoeffsType, Side>& h) { + using ResultType = typename internal::matrix_type_times_scalar_type<typename VectorsType::Scalar, OtherDerived>::Type; + eigen_assert(other.cols() == h.rows()); + ResultType res = ResultType::Identity(h.rows(), h.rows()); + h.applyThisOnTheLeft(res, true); + // The lazy diagonal product multiplies from the left, D(i,i) * H(i,j), as a noncommutative scalar requires, and + // reads each coefficient only to overwrite it, so it runs in place. + res = other.derived() * res; + return res; +} + +/** \brief Computes the product of a triangular or self-adjoint view with a Householder sequence. + * + * The view is evaluated into a dense matrix and the reflections are applied to it on the right, which has no + * structured path. + */ +template <typename OtherDerived, typename VectorsType, typename CoeffsType, int Side> +typename internal::matrix_type_times_scalar_type<typename VectorsType::Scalar, OtherDerived>::Type operator*( + const TriangularBase<OtherDerived>& other, const HouseholderSequence<VectorsType, CoeffsType, Side>& h) { + using ResultType = typename internal::matrix_type_times_scalar_type<typename VectorsType::Scalar, OtherDerived>::Type; + ResultType res; + internal::assign_householder_operand(res, other); + h.applyThisOnTheRight(res); + return res; +} + +/** \brief Computes the product of a permutation matrix with a Householder sequence. + * + * \f$ PH \f$ is \f$ H \f$ with its rows permuted: the sequence is evaluated and the permutation applied in place. + * The result takes the sequence's scalar type. + */ +template <typename OtherDerived, typename VectorsType, typename CoeffsType, int Side> +typename internal::permutation_type_as_matrix_type<typename VectorsType::Scalar, OtherDerived>::Type operator*( + const PermutationBase<OtherDerived>& other, const HouseholderSequence<VectorsType, CoeffsType, Side>& h) { + using ResultType = + typename internal::permutation_type_as_matrix_type<typename VectorsType::Scalar, OtherDerived>::Type; + eigen_assert(other.cols() == h.rows()); + ResultType res = ResultType::Identity(h.rows(), h.rows()); + h.applyThisOnTheLeft(res, true); + // The permutation product runs in place when its operand is the destination. + res.noalias() = other.derived() * res; + return res; +} + +/** \brief Computes the product of the inverse of a permutation matrix with a Householder sequence. */ +template <typename OtherDerived, typename VectorsType, typename CoeffsType, int Side> +typename internal::permutation_type_as_matrix_type<typename VectorsType::Scalar, Inverse<OtherDerived>>::Type operator*( + const InverseImpl<OtherDerived, PermutationStorage>& other, + const HouseholderSequence<VectorsType, CoeffsType, Side>& h) { + using ResultType = + typename internal::permutation_type_as_matrix_type<typename VectorsType::Scalar, Inverse<OtherDerived>>::Type; + eigen_assert(other.cols() == h.rows()); + ResultType res = ResultType::Identity(h.rows(), h.rows()); + h.applyThisOnTheLeft(res, true); + res.noalias() = other.derived() * res; + return res; +} + /** \ingroup Householder_Module * \householder_module * \brief Convenience function for constructing a Householder sequence.
diff --git a/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h b/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h index d14c71a..690d9dd 100644 --- a/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h +++ b/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h
@@ -280,6 +280,24 @@ return SolveWithGuess<Derived, Rhs, Guess>(derived(), b.derived(), x0); } + /** Solves \f$ A x = b \f$ in place: \a x holds the initial guess on entry and the solution on + * exit, and info(), iterations() and error() report on the run. + * + * solve() and solveWithGuess() return expressions and therefore require Eigen dense or sparse + * operands. This entry point accepts any single-column right-hand side and solution types that + * provide the vector operations the algorithm uses, for instance the device-resident vectors of + * the GPU module (\c contrib/Eigen/GPU) together with a matrix-free matrix type. + * + * \sa solve(), solveWithGuess() + */ + template <typename Rhs, typename Dest> + void solveWithGuessInPlace(const Rhs& b, Dest& x) const { + eigen_assert(m_isInitialized && "Solver is not initialized."); + eigen_assert(derived().rows() == b.rows() && + "solveWithGuessInPlace(): invalid number of rows of the right hand side b"); + derived()._solve_vector_with_guess_impl(b, x); + } + /** \returns Success if the iterations converged, and NoConvergence otherwise. */ ComputationInfo info() const { eigen_assert(m_isInitialized && "IterativeSolverBase is not initialized.");
diff --git a/Eigen/src/Jacobi/Jacobi.h b/Eigen/src/Jacobi/Jacobi.h index 8f90003..c4dac91 100644 --- a/Eigen/src/Jacobi/Jacobi.h +++ b/Eigen/src/Jacobi/Jacobi.h
@@ -109,17 +109,11 @@ // Scale this numerator/denominator pair by min(1, 1/|tau|), taking scale = 1 at tau = 0. // Then numerator <= 1 and denominator <= 1 + sqrt(2); only a ratio <= 1 is squared. // Normalize the pair directly to save the division forming |t|. - RealScalar numerator; - RealScalar denominator; - if (abs_delta > deno) { - const RealScalar ratio = deno / abs_delta; - numerator = ratio; - denominator = sqrt(RealScalar(1) + numext::abs2(ratio)) + RealScalar(1); - } else { - const RealScalar ratio = abs_delta / deno; - numerator = RealScalar(1); - denominator = sqrt(RealScalar(1) + numext::abs2(ratio)) + ratio; - } + // Select operands before dividing: speculative evaluation of both reciprocal ratios can overflow. + const bool large_delta = abs_delta > deno; + const RealScalar ratio = (large_delta ? deno : abs_delta) / (large_delta ? abs_delta : deno); + const RealScalar numerator = large_delta ? ratio : RealScalar(1); + const RealScalar denominator = sqrt(RealScalar(1) + numext::abs2(ratio)) + (large_delta ? RealScalar(1) : ratio); const RealScalar n = RealScalar(1) / sqrt(numext::abs2(numerator) + numext::abs2(denominator)); const RealScalar sine = numerator * n; const RealScalar sign_t = delta > RealScalar(0) ? RealScalar(1) : RealScalar(-1); @@ -532,7 +526,7 @@ // If d!=0, then t/d cannot overflow because the magnitude of the // entries forming d are not too small compared to the ones forming t. RealScalar u = t / d; - s1 = RealScalar(1) / numext::sqrt(RealScalar(1) + numext::abs2(u)); + s1 = RealScalar(1) / numext::sqrt<RealScalar>(RealScalar(1) + numext::abs2(u)); c1 = u * s1; }
diff --git a/Eigen/src/LU/PartialPivLU.h b/Eigen/src/LU/PartialPivLU.h index 06b5978..b9eb3f2 100644 --- a/Eigen/src/LU/PartialPivLU.h +++ b/Eigen/src/LU/PartialPivLU.h
@@ -349,12 +349,81 @@ namespace internal { +/** \internal Step K of the unblocked LU of a small fixed-size square matrix, with every block extent known at compile + * time. */ +template <int K, int Size, bool Last = (K == Size - 1)> +struct unrolled_partial_lu_step { + template <typename MatrixTypeRef, typename PivIndex> + static EIGEN_STRONG_INLINE void run(MatrixTypeRef& lu, PivIndex* row_transpositions, PivIndex& nb_transpositions, + Index& first_zero_pivot) { + using Scalar = typename MatrixTypeRef::Scalar; + using Scoring = scalar_score_coeff_op<Scalar>; + using Score = typename Scoring::result_type; + constexpr int Remaining = Size - K - 1; + + Index row_of_biggest_in_col; + const Score biggest_in_corner = + lu.col(K).template tail<Size - K>().unaryExpr(Scoring()).maxCoeff(&row_of_biggest_in_col); + row_of_biggest_in_col += K; + row_transpositions[K] = PivIndex(row_of_biggest_in_col); + if (!numext::is_exactly_zero(biggest_in_corner)) { + if (K != row_of_biggest_in_col) { + lu.row(K).swap(lu.row(row_of_biggest_in_col)); + ++nb_transpositions; + } + lu.col(K).template tail<Remaining>() /= lu.coeff(K, K); + } else if (first_zero_pivot == -1) { + first_zero_pivot = K; + } + lu.template bottomRightCorner<Remaining, Remaining>().noalias() -= + lu.col(K).template tail<Remaining>() * lu.row(K).template tail<Remaining>(); + unrolled_partial_lu_step<K + 1, Size>::run(lu, row_transpositions, nb_transpositions, first_zero_pivot); + } +}; + +template <int K, int Size> +struct unrolled_partial_lu_step<K, Size, /*Last=*/true> { + template <typename MatrixTypeRef, typename PivIndex> + static EIGEN_STRONG_INLINE void run(MatrixTypeRef& lu, PivIndex* row_transpositions, PivIndex&, + Index& first_zero_pivot) { + using Scoring = scalar_score_coeff_op<typename MatrixTypeRef::Scalar>; + row_transpositions[K] = PivIndex(K); + if (numext::is_exactly_zero(Scoring()(lu.coeff(K, K))) && first_zero_pivot == -1) first_zero_pivot = K; + } +}; + +/** \internal Runs the unrolled LU when Unroll is set; returns whether it did. */ +template <bool Unroll, int Size> +struct unrolled_partial_lu { + template <typename MatrixTypeRef, typename PivIndex> + static EIGEN_STRONG_INLINE bool run(MatrixTypeRef&, PivIndex*, PivIndex&, Index&) { + return false; + } +}; + +template <int Size> +struct unrolled_partial_lu<true, Size> { + template <typename MatrixTypeRef, typename PivIndex> + static EIGEN_STRONG_INLINE bool run(MatrixTypeRef& lu, PivIndex* row_transpositions, PivIndex& nb_transpositions, + Index& first_zero_pivot) { + nb_transpositions = 0; + first_zero_pivot = -1; + unrolled_partial_lu_step<0, Size>::run(lu, row_transpositions, nb_transpositions, first_zero_pivot); + return true; + } +}; + /** \internal This is the blocked version of unblocked_lu() */ template <typename Scalar, int StorageOrder, typename PivIndex, int SizeAtCompileTime = Dynamic> struct generic_partial_lu_impl { static constexpr int UnBlockedBound = 16; static constexpr bool UnBlockedAtCompileTime = SizeAtCompileTime != Dynamic && SizeAtCompileTime <= UnBlockedBound; static constexpr int ActualSizeAtCompileTime = UnBlockedAtCompileTime ? SizeAtCompileTime : Dynamic; + // Unrolling gives the pivot search, scaling and update of every step compile-time extents; runtime-sized blocks' + // vectorized loops cost more than their arithmetic (1.5-4.6x for 3x3 to 6x6 double, gcc and clang, AVX2). At 16, + // gcc's unrolled code is slower than the loop. + static constexpr int UnrolledBound = 12; + static constexpr bool UnrolledAtCompileTime = UnBlockedAtCompileTime && SizeAtCompileTime <= UnrolledBound; // Remaining rows and columns at compile-time: static constexpr int RRows = SizeAtCompileTime == 2 ? 1 : Dynamic; static constexpr int RCols = SizeAtCompileTime == 2 ? 1 : Dynamic; @@ -387,6 +456,12 @@ static Index unblocked_lu(MatrixTypeRef& lu, PivIndex* row_transpositions, PivIndex& nb_transpositions) { using Scoring = scalar_score_coeff_op<Scalar>; using Score = typename Scoring::result_type; + { + Index first_zero_pivot; + if (unrolled_partial_lu<UnrolledAtCompileTime, SizeAtCompileTime>::run(lu, row_transpositions, nb_transpositions, + first_zero_pivot)) + return first_zero_pivot; + } const Index rows = lu.rows(); const Index cols = lu.cols(); const Index size = (std::min)(rows, cols);
diff --git a/Eigen/src/SVD/BDCSVD.h b/Eigen/src/SVD/BDCSVD.h index cfaf3a5..3f0b0f0 100644 --- a/Eigen/src/SVD/BDCSVD.h +++ b/Eigen/src/SVD/BDCSVD.h
@@ -55,7 +55,9 @@ * #DisableQRDecomposition. It is not possible to request both the thin and full version of \a U or * \a V. By default, unitaries are not computed. BDCSVD uses R-Bidiagonalization to improve * performance on tall and wide matrices. For backwards compatibility, the option - * #DisableQRDecomposition can be used to disable this optimization. + * #DisableQRDecomposition can be used to disable this optimization. #PreconditionSquareMatrix is + * forwarded to the JacobiSVD that BDCSVD uses for inputs with fewer columns than the switch size, + * and does not affect larger inputs. * * This class first reduces the input matrix to bi-diagonal form using class UpperBidiagonalization, * and then performs a divide-and-conquer diagonalization. Small blocks are diagonalized using class JacobiSVD. @@ -233,7 +235,10 @@ void setSwitchSize(int s) { eigen_assert(s >= 3 && "BDCSVD the size of the algo switch has to be at least 3."); + if (s == m_impl.algoSwap()) return; m_impl.setAlgoSwap(s); + // smallSvd is only allocated for sizes below the switch, so the next compute must reallocate. + m_isAllocated = false; } private: @@ -255,7 +260,9 @@ void allocate_small(Index rows, Index cols, unsigned int computationOptions); internal::bdcsvd_impl<RealScalar> m_impl; bool m_isTranspose, m_useQrDecomp; - JacobiSVD<MatrixX> smallSvd; + // Only PreconditionSquareMatrix is forwarded: BDCSVD's QR bits configure its own R-bidiagonalization, and smallSvd + // needs its QR preconditioner to reduce non-square inputs. The unitaries are requested at runtime in allocate(). + JacobiSVD<MatrixX, (Options & int(PreconditionSquareMatrix))> smallSvd; HouseholderQR<MatrixX> qrDecomp; internal::UpperBidiagonalization<MatrixX> bid; MatrixX copyWorkspace; @@ -268,6 +275,7 @@ using Base::m_computeThinU; using Base::m_computeThinV; using Base::m_info; + using Base::m_isAllocated; using Base::m_isInitialized; using Base::m_matrixU; using Base::m_matrixV; @@ -284,7 +292,7 @@ if (Base::allocate(rows, cols, computationOptions)) return; if (cols < m_impl.algoSwap()) - smallSvd.allocate(rows, cols, Options == 0 ? computationOptions : internal::get_computation_options(Options)); + smallSvd.allocate(rows, cols, internal::get_computation_options(Options | computationOptions)); m_isTranspose = (cols > rows); @@ -315,7 +323,7 @@ void BDCSVD<MatrixType, Options>::allocate_small(Index rows, Index cols, unsigned int computationOptions) { if (Base::allocate(rows, cols, computationOptions)) return; - smallSvd.allocate(rows, cols, Options == 0 ? computationOptions : internal::get_computation_options(Options)); + smallSvd.allocate(rows, cols, internal::get_computation_options(Options | computationOptions)); m_isTranspose = (cols > rows); } @@ -375,7 +383,10 @@ } //**** step 0 - Copy the input matrix and apply scaling to reduce over/under-flows - const RealScalar maxCoeff = matrix.cwiseAbs().template maxCoeff<PropagateNaN>(); + // A SIMD unit that flushes subnormal inputs reads an all-subnormal matrix as zero; recover its maximum from the + // representation so that the scaling still brings it into the normal range. + const RealScalar maxCoeff = internal::safe_scaling<RealScalar>::recover_flushed_max_coeff( + matrix.derived(), matrix.cwiseAbs().template maxCoeff<PropagateNaN>()); if (!(numext::isfinite)(maxCoeff)) { m_isInitialized = true; m_info = InvalidInput; @@ -422,7 +433,7 @@ //**** step 3 - Copy singular values and vectors for (int i = 0; i < diagSize(); i++) { RealScalar a = abs(m_impl.computed().coeff(i, i)); - internal::safe_scaling<RealScalar>::unscale_to(m_singularValues.coeffRef(i), a, factors); + m_singularValues.coeffRef(i) = a; if (a < considerZero) { m_nonzeroSingularValues = i; m_singularValues.tail(diagSize() - i - 1).setZero(); @@ -432,6 +443,8 @@ break; } } + // Unscaling with maxCoeff keeps singular values that land in the subnormal range under FTZ. + internal::safe_scaling<RealScalar>::unscale_in_place(m_singularValues, maxCoeff, factors); //**** step 4 - Finalize unitaries U and V if (m_isTranspose) @@ -510,10 +523,14 @@ return *this; } - // Check for non-finite inputs. - const RealScalar diagScale = diagonal.cwiseAbs().template maxCoeff<PropagateNaN>(); - const RealScalar superdiagScale = n > 1 ? superdiagonal.cwiseAbs().template maxCoeff<PropagateNaN>() : RealScalar(0); - const RealScalar maxCoeff = numext::maxi(diagScale, superdiagScale); + // Check for non-finite inputs. The rescan recovers an all-subnormal input that a flushing SIMD unit reads as zero. + const RealScalar diagScale = internal::safe_scaling<RealScalar>::recover_flushed_max_coeff( + diagonal.derived(), diagonal.cwiseAbs().template maxCoeff<PropagateNaN>()); + const RealScalar superdiagScale = + n > 1 ? internal::safe_scaling<RealScalar>::recover_flushed_max_coeff( + superdiagonal.derived(), superdiagonal.cwiseAbs().template maxCoeff<PropagateNaN>()) + : RealScalar(0); + const RealScalar maxCoeff = internal::max_preserving_subnormals(diagScale, superdiagScale); if (!(numext::isfinite)(maxCoeff)) { m_isInitialized = true; m_info = InvalidInput; @@ -537,7 +554,7 @@ m_isInitialized = true; m_info = smallSvd.info(); if (m_info == Success || m_info == NoConvergence) { - internal::safe_scaling<RealScalar>::unscale_to(m_singularValues, smallSvd.singularValues(), factors); + internal::safe_scaling<RealScalar>::unscale_to(m_singularValues, smallSvd.singularValues(), maxCoeff, factors); m_nonzeroSingularValues = smallSvd.nonzeroSingularValues(); if (computeU()) m_matrixU = smallSvd.matrixU(); if (computeV()) m_matrixV = smallSvd.matrixV(); @@ -574,7 +591,7 @@ //**** Extract singular values. for (int i = 0; i < diagSize(); i++) { RealScalar a = abs(m_impl.computed().coeff(i, i)); - internal::safe_scaling<RealScalar>::unscale_to(m_singularValues.coeffRef(i), a, factors); + m_singularValues.coeffRef(i) = a; if (a < considerZero) { m_nonzeroSingularValues = i; m_singularValues.tail(diagSize() - i - 1).setZero(); @@ -584,6 +601,8 @@ break; } } + // Unscaling with maxCoeff keeps singular values that land in the subnormal range under FTZ. + internal::safe_scaling<RealScalar>::unscale_in_place(m_singularValues, maxCoeff, factors); //**** Copy U and V directly (no Householder to apply). // D&C computes B^T = naiveU * S * naiveV^T, so B = naiveV * S * naiveU^T.
diff --git a/Eigen/src/SVD/JacobiSVD.h b/Eigen/src/SVD/JacobiSVD.h index 9120423..d1510a0 100644 --- a/Eigen/src/SVD/JacobiSVD.h +++ b/Eigen/src/SVD/JacobiSVD.h
@@ -33,7 +33,12 @@ enum { PreconditionIfMoreColsThanRows, PreconditionIfMoreRowsThanCols }; -template <typename MatrixType, int QRPreconditioner, int Case> +// If option PreconditionSquareMatrix is set, square matrices take the PreconditionIfMoreRowsThanCols path: A = Q R P^*. +constexpr bool svd_precondition_more_rows(int options, Index rows, Index cols) { + return rows > cols || (should_svd_precondition_square_matrix(options) && rows == cols); +} + +template <typename MatrixType, int QRPreconditioner, int Case, bool PreconditionSquare> struct qr_preconditioner_should_do_anything : bool_constant<!((QRPreconditioner == NoQRPreconditioner) || (Case == PreconditionIfMoreColsThanRows && MatrixType::RowsAtCompileTime != Dynamic && @@ -41,10 +46,12 @@ MatrixType::ColsAtCompileTime <= MatrixType::RowsAtCompileTime) || (Case == PreconditionIfMoreRowsThanCols && MatrixType::RowsAtCompileTime != Dynamic && MatrixType::ColsAtCompileTime != Dynamic && - MatrixType::RowsAtCompileTime <= MatrixType::ColsAtCompileTime))> {}; + (PreconditionSquare ? MatrixType::RowsAtCompileTime < MatrixType::ColsAtCompileTime + : MatrixType::RowsAtCompileTime <= MatrixType::ColsAtCompileTime)))> {}; template <typename MatrixType, int Options, int QRPreconditioner, int Case, - bool DoAnything = qr_preconditioner_should_do_anything<MatrixType, QRPreconditioner, Case>::value> + bool DoAnything = qr_preconditioner_should_do_anything<MatrixType, QRPreconditioner, Case, + should_svd_precondition_square_matrix(Options)>::value> struct qr_preconditioner_impl {}; template <typename MatrixType, int Options, int QRPreconditioner, int Case> @@ -79,7 +86,7 @@ } template <typename Xpr> bool run(SVDType& svd, const Xpr& matrix) { - if (matrix.rows() > matrix.cols()) { + if (svd_precondition_more_rows(Options, matrix.rows(), matrix.cols())) { m_qr.compute(matrix); svd.m_workMatrix = m_qr.matrixQR().block(0, 0, matrix.cols(), matrix.cols()).template triangularView<Upper>(); if (svd.m_computeFullU) m_qr.matrixQ().evalTo(svd.m_matrixU, m_workspace); @@ -168,7 +175,7 @@ } template <typename Xpr> bool run(SVDType& svd, const Xpr& matrix) { - if (matrix.rows() > matrix.cols()) { + if (svd_precondition_more_rows(Options, matrix.rows(), matrix.cols())) { m_qr.compute(matrix); svd.m_workMatrix = m_qr.matrixQR().block(0, 0, matrix.cols(), matrix.cols()).template triangularView<Upper>(); if (svd.m_computeFullU) @@ -274,7 +281,7 @@ } template <typename Xpr> bool run(SVDType& svd, const Xpr& matrix) { - if (matrix.rows() > matrix.cols()) { + if (svd_precondition_more_rows(Options, matrix.rows(), matrix.cols())) { m_qr.compute(matrix); svd.m_workMatrix = m_qr.matrixQR().block(0, 0, matrix.cols(), matrix.cols()).template triangularView<Upper>(); if (svd.m_computeFullU) @@ -539,8 +546,9 @@ * * \tparam MatrixType_ the type of the matrix of which we are computing the SVD decomposition * \tparam Options this optional parameter allows one to specify the type of QR decomposition that will be used - * internally for the R-SVD step for non-square matrices. Additionally, it allows one to specify whether to compute thin - * or full unitaries \a U and \a V. See discussion of possible values below. + * internally for the R-SVD step on rectangular matrices. The R-SVD step can be applied to square matrices using + * #PreconditionSquareMatrix. Additionally, it allows one to specify whether to compute thin or full unitaries \a U and + * \a V. See discussion of possible values below. * * SVD decomposition consists in decomposing any n-by-p matrix \a A as a product * \f[ A = U S V^* \f] @@ -586,12 +594,18 @@ * significantly speed up computation, since JacobiSVD is always checking if QR preconditioning is needed before * applying it anyway. * + * By default, square matrices are not QR preconditioned. Adding #PreconditionSquareMatrix to the Options template + * parameter applies the selected QR preconditioner to square matrices too: the Jacobi iteration then runs on the + * triangular factor \a R of \f$ A P = Q R \f$ instead of on \a A. This can be a performance improvement for some + * matrices because it can reduce the total number of iterations. #PreconditionSquareMatrix cannot be combined with + * #NoQRPreconditioner. + * * One may also use the Options template parameter to specify how the unitaries should be computed. The options are * #ComputeThinU, #ComputeThinV, #ComputeFullU, #ComputeFullV. It is not possible to request both the thin and full * versions of a unitary. By default, unitaries will not be computed. * * You can set the QRPreconditioner and unitary options together: JacobiSVD<MatrixType, - * ColPivHouseholderQRPreconditioner | ComputeThinU | ComputeFullV> + * ColPivHouseholderQRPreconditioner | PreconditionSquareMatrix | ComputeThinU | ComputeFullV> * * \sa MatrixBase::jacobiSvd() */ @@ -733,14 +747,14 @@ void allocate(Index rows_, Index cols_, unsigned int computationOptions) { if (Base::allocate(rows_, cols_, computationOptions)) return; - eigen_assert(!(ShouldComputeThinU && int(QRPreconditioner) == int(FullPivHouseholderQRPreconditioner)) && - !(ShouldComputeThinU && int(QRPreconditioner) == int(FullPivHouseholderQRPreconditioner)) && - "JacobiSVD: can't compute thin U or thin V with the FullPivHouseholderQR preconditioner. " - "Use the ColPivHouseholderQR preconditioner instead."); + eigen_assert( + !((m_computeThinU || m_computeThinV) && int(QRPreconditioner) == int(FullPivHouseholderQRPreconditioner)) && + "JacobiSVD: can't compute thin U or thin V with the FullPivHouseholderQR preconditioner. " + "Use the ColPivHouseholderQR preconditioner instead."); m_workMatrix.resize(diagSize(), diagSize()); if (cols() > rows()) m_qr_precond_morecols.allocate(*this); - if (rows() > cols()) m_qr_precond_morerows.allocate(*this); + if (internal::svd_precondition_more_rows(Options, rows(), cols())) m_qr_precond_morerows.allocate(*this); } private: @@ -774,9 +788,12 @@ using Base::ShouldComputeThinV; EIGEN_STATIC_ASSERT(!(ShouldComputeThinU && int(QRPreconditioner) == int(FullPivHouseholderQRPreconditioner)) && - !(ShouldComputeThinU && int(QRPreconditioner) == int(FullPivHouseholderQRPreconditioner)), + !(ShouldComputeThinV && int(QRPreconditioner) == int(FullPivHouseholderQRPreconditioner)), "JacobiSVD: can't compute thin U or thin V with the FullPivHouseholderQR preconditioner. " "Use the ColPivHouseholderQR preconditioner instead.") + EIGEN_STATIC_ASSERT(!(internal::should_svd_precondition_square_matrix(Options) && + int(QRPreconditioner) == int(NoQRPreconditioner)), + "JacobiSVD: PreconditionSquareMatrix requires a QR preconditioner other than NoQRPreconditioner.") template <typename MatrixType__, int Options__, bool IsComplex_> friend struct internal::svd_precondition_2x2_block_to_be_real; @@ -826,8 +843,10 @@ // limit for denormal numbers to be considered zero in order to avoid infinite loops (see bug 286) const RealScalar considerAsZero = (std::numeric_limits<RealScalar>::min)(); - // Scaling factor to reduce over/under-flows - const RealScalar maxCoeff = matrix.cwiseAbs().template maxCoeff<PropagateNaN>(); + // Scaling factor to reduce over/under-flows. A SIMD unit that flushes subnormal inputs reads an all-subnormal matrix + // as zero; recover its maximum from the representation so that the scaling still brings it into the normal range. + const RealScalar maxCoeff = internal::safe_scaling<RealScalar>::recover_flushed_max_coeff( + matrix.derived(), matrix.cwiseAbs().template maxCoeff<PropagateNaN>()); if (!(numext::isfinite)(maxCoeff)) { m_isInitialized = true; m_info = InvalidInput; @@ -839,7 +858,7 @@ /*** step 1. The R-SVD step: we use a QR decomposition to reduce to the case of a square matrix */ - if (rows() != cols()) { + if (rows() != cols() || internal::should_svd_precondition_square_matrix(Options)) { factors = internal::safe_scaling<RealScalar>::with_scaled(matrix.derived(), maxCoeff, [&](const auto& scaledMatrix) { m_qr_precond_morecols.run(*this, scaledMatrix); @@ -907,24 +926,25 @@ m_singularValues.coeffRef(i) = abs(a); if (computeU()) m_matrixU.col(i) *= m_workMatrix.coeff(i, i) / a; } else { - // m_workMatrix.coeff(i,i) is already real, no difficulty: + // m_workMatrix.coeff(i,i) is already real. Its magnitude and sign are read from the representation for float + // and double: a subnormal entry compares as zero under DAZ, and an abs() that widens flushes it under FTZ. RealScalar a = numext::real(m_workMatrix.coeff(i, i)); - m_singularValues.coeffRef(i) = abs(a); - if (computeU() && (a < RealScalar(0))) m_matrixU.col(i) = -m_matrixU.col(i); + m_singularValues.coeffRef(i) = internal::abs_preserving_subnormals(a); + if (computeU() && internal::is_negative_preserving_subnormals(a)) m_matrixU.col(i) = -m_matrixU.col(i); } } - internal::safe_scaling<RealScalar>::unscale_in_place(m_singularValues, factors); - /*** step 4. Sort singular values in descending order and compute the number of nonzero singular values ***/ - m_nonzeroSingularValues = diagSize(); + // Sort in the scaled frame, where the largest values are normal. A tail whose maximum reads zero or subnormal + // holds zeros and subnormals, which FTZ/DAZ hardware compares as zero: order it from the representation. for (Index i = 0; i < diagSize(); i++) { Index pos; RealScalar maxRemainingSingularValue = m_singularValues.tail(diagSize() - i).maxCoeff(&pos); - if (numext::is_exactly_zero(maxRemainingSingularValue)) { - m_nonzeroSingularValues = i; - break; + if (internal::is_zero_or_subnormal_magnitude(maxRemainingSingularValue)) { + pos = internal::safe_scaling<RealScalar>::recover_flushed_max_coeff_index(m_singularValues.tail(diagSize() - i), + pos); + if (numext::is_exactly_zero_no_flush(m_singularValues.coeff(i + pos))) break; } if (pos) { pos += i; @@ -933,6 +953,16 @@ if (computeV()) m_matrixV.col(pos).swap(m_matrixV.col(i)); } } + // Unscaling with maxCoeff keeps singular values that land in the subnormal range under FTZ; the count then reads + // them from the representation. + internal::safe_scaling<RealScalar>::unscale_in_place(m_singularValues, maxCoeff, factors); + m_nonzeroSingularValues = diagSize(); + for (Index i = 0; i < diagSize(); i++) { + if (numext::is_exactly_zero_no_flush(m_singularValues.coeff(i))) { + m_nonzeroSingularValues = i; + break; + } + } m_isInitialized = true; return *this;
diff --git a/Eigen/src/SVD/SVDBase.h b/Eigen/src/SVD/SVDBase.h index c224c93..cdc59c6 100644 --- a/Eigen/src/SVD/SVDBase.h +++ b/Eigen/src/SVD/SVDBase.h
@@ -34,6 +34,8 @@ constexpr int get_computation_options(int options) { return options & ComputationOptionsBits; } +constexpr bool should_svd_precondition_square_matrix(int options) { return (options & PreconditionSquareMatrix) != 0; } + constexpr bool should_svd_compute_thin_u(int options) { return (options & ComputeThinU) != 0; } constexpr bool should_svd_compute_full_u(int options) { return (options & ComputeFullU) != 0; } constexpr bool should_svd_compute_thin_v(int options) { return (options & ComputeThinV) != 0; }
diff --git a/Eigen/src/SparseCore/SparseFuzzy.h b/Eigen/src/SparseCore/SparseApproxComparisons.h similarity index 89% rename from Eigen/src/SparseCore/SparseFuzzy.h rename to Eigen/src/SparseCore/SparseApproxComparisons.h index 5701778..8db78d6 100644 --- a/Eigen/src/SparseCore/SparseFuzzy.h +++ b/Eigen/src/SparseCore/SparseApproxComparisons.h
@@ -8,8 +8,8 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. // SPDX-License-Identifier: MPL-2.0 -#ifndef EIGEN_SPARSE_FUZZY_H -#define EIGEN_SPARSE_FUZZY_H +#ifndef EIGEN_SPARSE_APPROX_COMPARISONS_H +#define EIGEN_SPARSE_APPROX_COMPARISONS_H // IWYU pragma: private #include "./InternalHeaderCheck.h" @@ -29,4 +29,4 @@ } // end namespace Eigen -#endif // EIGEN_SPARSE_FUZZY_H +#endif // EIGEN_SPARSE_APPROX_COMPARISONS_H
diff --git a/Eigen/src/plugins/ArrayCwiseUnaryOps.inc b/Eigen/src/plugins/ArrayCwiseUnaryOps.inc index 4bfbd46..9718e71 100644 --- a/Eigen/src/plugins/ArrayCwiseUnaryOps.inc +++ b/Eigen/src/plugins/ArrayCwiseUnaryOps.inc
@@ -561,6 +561,13 @@ * This function computes the coefficient-wise power. The function MatrixBase::pow() in the * contrib module MatrixFunctions computes the matrix power. * + * For a complex<float> or complex<double> base and an integer-valued exponent \a n other than 0 and 1, zero and + * infinite bases are the points 0 and infinity of the extended complex plane, with every value that has an infinite + * component, a NaN one included, the infinity +inf + 0i that std::proj returns: a zero base gives +0 + 0i for + * \a n > 1 and +inf + 0i for \a n < 0, an infinite base the reverse, whatever the signs of their zeros. A base with a + * NaN component and no infinite one gives NaN + NaN i. \a n = 1 returns the base itself. std::pow leaves all of these + * to the implementation. + * * Example: \include Cwise_pow.cpp * Output: \verbinclude Cwise_pow.out *
diff --git a/blas/BandTriangularSolver.h b/blas/BandTriangularSolver.h index 7af5684..07339a1 100644 --- a/blas/BandTriangularSolver.h +++ b/blas/BandTriangularSolver.h
@@ -32,11 +32,11 @@ const LhsMap&> cjLhs(lhs); - for (int col = 0; col < other.cols(); ++col) { - for (int ii = 0; ii < size; ++ii) { - int i = IsLower ? ii : size - ii - 1; - int actual_k = (std::min)(k, ii); - int actual_start = IsLower ? k - actual_k : 1; + for (Index col = 0; col < other.cols(); ++col) { + for (Index ii = 0; ii < size; ++ii) { + Index i = IsLower ? ii : size - ii - 1; + Index actual_k = (std::min)(k, ii); + Index actual_start = IsLower ? k - actual_k : 1; if (actual_k > 0) other.coeffRef(i, col) -= cjLhs.row(i) @@ -63,11 +63,11 @@ const LhsMap&> cjLhs(lhs); - for (int col = 0; col < other.cols(); ++col) { - for (int ii = 0; ii < size; ++ii) { - int i = IsLower ? ii : size - ii - 1; - int actual_k = (std::min)(k, size - ii - 1); - int actual_start = IsLower ? 1 : k - actual_k; + for (Index col = 0; col < other.cols(); ++col) { + for (Index ii = 0; ii < size; ++ii) { + Index i = IsLower ? ii : size - ii - 1; + Index actual_k = (std::min)(k, size - ii - 1); + Index actual_start = IsLower ? 1 : k - actual_k; EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) other.coeffRef(i, col) /= cjLhs(IsLower ? 0 : k, i);
diff --git a/blas/CMakeLists.txt b/blas/CMakeLists.txt index 4053b03..c7f45e6 100644 --- a/blas/CMakeLists.txt +++ b/blas/CMakeLists.txt
@@ -25,6 +25,29 @@ list(APPEND EIGEN_BLAS_TARGETS eigen_blas) endif() +if(CMAKE_SIZEOF_VOID_P EQUAL 8 AND NOT EIGEN_TEST_32BIT) + option(EIGEN_BUILD_BLAS_ILP64 "Toggles the building of the 64-bit integer (ILP64) Eigen Blas library" ON) +else() + set(EIGEN_BUILD_BLAS_ILP64 OFF) +endif() + +if(EIGEN_BUILD_BLAS_ILP64) + add_library(eigen_blas_ilp64_static STATIC ${EigenBlas_SRCS}) + target_compile_definitions(eigen_blas_ilp64_static PUBLIC "EIGEN_64BIT_BLAS") + list(APPEND EIGEN_BLAS_TARGETS eigen_blas_ilp64_static) + + if(EIGEN_BUILD_SHARED_LIBS) + add_library(eigen_blas_ilp64 SHARED ${EigenBlas_SRCS} "eigen_blas.def") + # EIGEN_BLAS_BUILD_DLL stays private: a consumer that sees it declares the routines dllexport, + # which MSVC rejects against the plain declarations EIGEN_USE_BLAS brings in (C2375). + target_compile_definitions(eigen_blas_ilp64 PRIVATE "EIGEN_BLAS_BUILD_DLL" PUBLIC "EIGEN_64BIT_BLAS") + set_target_properties(eigen_blas_ilp64 PROPERTIES CXX_VISIBILITY_PRESET hidden + VERSION ${EIGEN_VERSION_NUMBER} + SOVERSION ${EIGEN_MAJOR_VERSION}) + list(APPEND EIGEN_BLAS_TARGETS eigen_blas_ilp64) + endif() +endif() + foreach(target IN LISTS EIGEN_BLAS_TARGETS) if(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO) target_link_libraries(${target} ${EIGEN_STANDARD_LIBRARIES_TO_LINK_TO})
diff --git a/blas/README.txt b/blas/README.txt index 63a5203..c744328 100644 --- a/blas/README.txt +++ b/blas/README.txt
@@ -4,3 +4,7 @@ This module is not built by default. In order to compile it, you need to type 'make blas' from within your build dir. +On 64-bit platforms, 'make blas' builds both 32-bit integer (LP64: eigen_blas, +eigen_blas_static) and 64-bit integer (ILP64: eigen_blas_ilp64, +eigen_blas_ilp64_static) libraries by default (controlled by +EIGEN_BUILD_BLAS_ILP64).
diff --git a/blas/blas.h b/blas/blas.h index 51fbcf4..6d5c5bf 100644 --- a/blas/blas.h +++ b/blas/blas.h
@@ -5,6 +5,7 @@ #define BLAS_H #include <stddef.h> +#include <stdint.h> #if defined(_WIN32) #if defined(EIGEN_BLAS_BUILD_DLL) @@ -24,7 +25,21 @@ extern "C" { #endif -#define BLASFUNC(FUNC) FUNC##_ +#ifndef BLAS_FUNC_SUFFIX +#define BLAS_FUNC_SUFFIX _ +#endif + +#define CONCAT_EXPAND(a, b) a##b +#define CONCAT(a, b) CONCAT_EXPAND(a, b) +#define BLASFUNC(FUNC) CONCAT(FUNC, BLAS_FUNC_SUFFIX) + +#ifndef EIGEN_BLAS_INT +#if defined(EIGEN_64BIT_BLAS) +#define EIGEN_BLAS_INT int64_t +#else +#define EIGEN_BLAS_INT int +#endif +#endif #ifdef __WIN64__ typedef long long BLASLONG; @@ -35,149 +50,168 @@ #endif EIGEN_BLAS_API int BLASFUNC(lsame)(const char *, const char *); -EIGEN_BLAS_API void BLASFUNC(xerbla)(const char*, int* info, size_t len); +EIGEN_BLAS_API void BLASFUNC(xerbla)(const char *, EIGEN_BLAS_INT *info, size_t len); -EIGEN_BLAS_API float BLASFUNC(sdot)(int *, float *, int *, float *, int *); -EIGEN_BLAS_API float BLASFUNC(sdsdot)(int *, float *, float *, int *, float *, int *); +EIGEN_BLAS_API float BLASFUNC(sdot)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API float BLASFUNC(sdsdot)(EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API double BLASFUNC(dsdot)(int *, float *, int *, float *, int *); -EIGEN_BLAS_API double BLASFUNC(ddot)(int *, double *, int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qdot)(int *, double *, int *, double *, int *); +EIGEN_BLAS_API double BLASFUNC(dsdot)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(ddot)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qdot)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(cdotu)(int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(cdotc)(int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zdotu)(int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(zdotc)(int *, double *, int *, double *, int *); +/* +EIGEN_BLAS_API void BLASFUNC(cdotu)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cdotc)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zdotu)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zdotc)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +*/ -EIGEN_BLAS_API void BLASFUNC(cdotuw)(int *, float *, int *, float *, int *, float *); -EIGEN_BLAS_API void BLASFUNC(cdotcw)(int *, float *, int *, float *, int *, float *); -EIGEN_BLAS_API void BLASFUNC(zdotuw)(int *, double *, int *, double *, int *, double *); -EIGEN_BLAS_API void BLASFUNC(zdotcw)(int *, double *, int *, double *, int *, double *); +EIGEN_BLAS_API void BLASFUNC(cdotuw)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *); +EIGEN_BLAS_API void BLASFUNC(cdotcw)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *); +EIGEN_BLAS_API void BLASFUNC(zdotuw)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, + double *); +EIGEN_BLAS_API void BLASFUNC(zdotcw)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, + double *); -EIGEN_BLAS_API void BLASFUNC(saxpy)(const int *, const float *, const float *, const int *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(daxpy)(const int *, const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(qaxpy)(const int *, const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(caxpy)(const int *, const float *, const float *, const int *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zaxpy)(const int *, const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xaxpy)(const int *, const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(caxpyc)(const int *, const float *, const float *, const int *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zaxpyc)(const int *, const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xaxpyc)(const int *, const double *, const double *, const int *, double *, const int *); +EIGEN_BLAS_API void BLASFUNC(saxpy)(const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(daxpy)(const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qaxpy)(const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(caxpy)(const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zaxpy)(const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xaxpy)(const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(caxpyc)(const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zaxpyc)(const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xaxpyc)(const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(saxpby)(const int *, const float *, const float *, const int *, const float *, float *, - const int *); -EIGEN_BLAS_API void BLASFUNC(daxpby)(const int *, const double *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(qaxpby)(const int *, const double *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(caxpby)(const int *, const float *, const float *, const int *, const float *, float *, - const int *); -EIGEN_BLAS_API void BLASFUNC(zaxpby)(const int *, const double *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(xaxpby)(const int *, const double *, const double *, const int *, const double *, double *, - const int *); +EIGEN_BLAS_API void BLASFUNC(saxpby)(const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + const float *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(daxpby)(const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qaxpby)(const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(caxpby)(const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + const float *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zaxpby)(const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xaxpby)(const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(scopy)(int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dcopy)(int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(qcopy)(int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(ccopy)(int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zcopy)(int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xcopy)(int *, double *, int *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(scopy)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dcopy)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qcopy)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ccopy)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zcopy)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xcopy)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(sswap)(int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dswap)(int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(qswap)(int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(cswap)(int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zswap)(int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xswap)(int *, double *, int *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(sswap)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dswap)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qswap)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cswap)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zswap)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xswap)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API float BLASFUNC(sasum)(int *, float *, int *); -EIGEN_BLAS_API float BLASFUNC(scasum)(int *, float *, int *); -EIGEN_BLAS_API double BLASFUNC(dasum)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qasum)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(dzasum)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qxasum)(int *, double *, int *); +EIGEN_BLAS_API float BLASFUNC(sasum)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API float BLASFUNC(scasum)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(dasum)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qasum)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(dzasum)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qxasum)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API int BLASFUNC(isamax)(int *, float *, int *); -EIGEN_BLAS_API int BLASFUNC(idamax)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(iqamax)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(icamax)(int *, float *, int *); -EIGEN_BLAS_API int BLASFUNC(izamax)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(ixamax)(int *, double *, int *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(isamax)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(idamax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(iqamax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(icamax)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(izamax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(ixamax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API int BLASFUNC(ismax)(int *, float *, int *); -EIGEN_BLAS_API int BLASFUNC(idmax)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(iqmax)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(icmax)(int *, float *, int *); -EIGEN_BLAS_API int BLASFUNC(izmax)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(ixmax)(int *, double *, int *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(ismax)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(idmax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(iqmax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(icmax)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(izmax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(ixmax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API int BLASFUNC(isamin)(int *, float *, int *); -EIGEN_BLAS_API int BLASFUNC(idamin)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(iqamin)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(icamin)(int *, float *, int *); -EIGEN_BLAS_API int BLASFUNC(izamin)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(ixamin)(int *, double *, int *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(isamin)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(idamin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(iqamin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(icamin)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(izamin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(ixamin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API int BLASFUNC(ismin)(int *, float *, int *); -EIGEN_BLAS_API int BLASFUNC(idmin)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(iqmin)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(icmin)(int *, float *, int *); -EIGEN_BLAS_API int BLASFUNC(izmin)(int *, double *, int *); -EIGEN_BLAS_API int BLASFUNC(ixmin)(int *, double *, int *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(ismin)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(idmin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(iqmin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(icmin)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(izmin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API EIGEN_BLAS_INT BLASFUNC(ixmin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API float BLASFUNC(samax)(int *, float *, int *); -EIGEN_BLAS_API double BLASFUNC(damax)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qamax)(int *, double *, int *); -EIGEN_BLAS_API float BLASFUNC(scamax)(int *, float *, int *); -EIGEN_BLAS_API double BLASFUNC(dzamax)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qxamax)(int *, double *, int *); +EIGEN_BLAS_API float BLASFUNC(samax)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(damax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qamax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API float BLASFUNC(scamax)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(dzamax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qxamax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API float BLASFUNC(samin)(int *, float *, int *); -EIGEN_BLAS_API double BLASFUNC(damin)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qamin)(int *, double *, int *); -EIGEN_BLAS_API float BLASFUNC(scamin)(int *, float *, int *); -EIGEN_BLAS_API double BLASFUNC(dzamin)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qxamin)(int *, double *, int *); +EIGEN_BLAS_API float BLASFUNC(samin)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(damin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qamin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API float BLASFUNC(scamin)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(dzamin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qxamin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API float BLASFUNC(smax)(int *, float *, int *); -EIGEN_BLAS_API double BLASFUNC(dmax)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qmax)(int *, double *, int *); -EIGEN_BLAS_API float BLASFUNC(scmax)(int *, float *, int *); -EIGEN_BLAS_API double BLASFUNC(dzmax)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qxmax)(int *, double *, int *); +EIGEN_BLAS_API float BLASFUNC(smax)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(dmax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qmax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API float BLASFUNC(scmax)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(dzmax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qxmax)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API float BLASFUNC(smin)(int *, float *, int *); -EIGEN_BLAS_API double BLASFUNC(dmin)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qmin)(int *, double *, int *); -EIGEN_BLAS_API float BLASFUNC(scmin)(int *, float *, int *); -EIGEN_BLAS_API double BLASFUNC(dzmin)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qxmin)(int *, double *, int *); +EIGEN_BLAS_API float BLASFUNC(smin)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(dmin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qmin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API float BLASFUNC(scmin)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(dzmin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qxmin)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(sscal)(int *, float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dscal)(int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(qscal)(int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(cscal)(int *, float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zscal)(int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xscal)(int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(csscal)(int *, float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zdscal)(int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xqscal)(int *, double *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(sscal)(EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dscal)(EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qscal)(EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cscal)(EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zscal)(EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xscal)(EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(csscal)(EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zdscal)(EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xqscal)(EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API float BLASFUNC(snrm2)(int *, float *, int *); -EIGEN_BLAS_API float BLASFUNC(scnrm2)(int *, float *, int *); +EIGEN_BLAS_API float BLASFUNC(snrm2)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API float BLASFUNC(scnrm2)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API double BLASFUNC(dnrm2)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qnrm2)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(dznrm2)(int *, double *, int *); -EIGEN_BLAS_API double BLASFUNC(qxnrm2)(int *, double *, int *); +EIGEN_BLAS_API double BLASFUNC(dnrm2)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qnrm2)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(dznrm2)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API double BLASFUNC(qxnrm2)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(srot)(int *, float *, int *, float *, int *, float *, float *); -EIGEN_BLAS_API void BLASFUNC(drot)(int *, double *, int *, double *, int *, double *, double *); -EIGEN_BLAS_API void BLASFUNC(qrot)(int *, double *, int *, double *, int *, double *, double *); -EIGEN_BLAS_API void BLASFUNC(csrot)(int *, float *, int *, float *, int *, float *, float *); -EIGEN_BLAS_API void BLASFUNC(zdrot)(int *, double *, int *, double *, int *, double *, double *); -EIGEN_BLAS_API void BLASFUNC(xqrot)(int *, double *, int *, double *, int *, double *, double *); +EIGEN_BLAS_API void BLASFUNC(srot)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, + float *); +EIGEN_BLAS_API void BLASFUNC(drot)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, + double *); +EIGEN_BLAS_API void BLASFUNC(qrot)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, + double *); +EIGEN_BLAS_API void BLASFUNC(csrot)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, + float *); +EIGEN_BLAS_API void BLASFUNC(zdrot)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, + double *); +EIGEN_BLAS_API void BLASFUNC(xqrot)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, + double *); EIGEN_BLAS_API void BLASFUNC(srotg)(float *, float *, float *, float *); EIGEN_BLAS_API void BLASFUNC(drotg)(double *, double *, double *, double *); @@ -189,372 +223,489 @@ EIGEN_BLAS_API void BLASFUNC(srotmg)(float *, float *, float *, float *, float *); EIGEN_BLAS_API void BLASFUNC(drotmg)(double *, double *, double *, double *, double *); -EIGEN_BLAS_API void BLASFUNC(srotm)(int *, float *, int *, float *, int *, float *); -EIGEN_BLAS_API void BLASFUNC(drotm)(int *, double *, int *, double *, int *, double *); -EIGEN_BLAS_API void BLASFUNC(qrotm)(int *, double *, int *, double *, int *, double *); +EIGEN_BLAS_API void BLASFUNC(srotm)(EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *); +EIGEN_BLAS_API void BLASFUNC(drotm)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *); +EIGEN_BLAS_API void BLASFUNC(qrotm)(EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *); /* Level 2 routines */ -EIGEN_BLAS_API void BLASFUNC(sger)(int *, int *, float *, float *, int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dger)(int *, int *, double *, double *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(qger)(int *, int *, double *, double *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(cgeru)(int *, int *, float *, float *, int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(cgerc)(int *, int *, float *, float *, int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zgeru)(int *, int *, double *, double *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(zgerc)(int *, int *, double *, double *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xgeru)(int *, int *, double *, double *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xgerc)(int *, int *, double *, double *, int *, double *, int *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(sger)(EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dger)(EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qger)(EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cgeru)(EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cgerc)(EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zgeru)(EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zgerc)(EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xgeru)(EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xgerc)(EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(sgemv)(const char *, const int *, const int *, const float *, const float *, const int *, - const float *, const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(dgemv)(const char *, const int *, const int *, const double *, const double *, const int *, - const double *, const int *, const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(qgemv)(const char *, const int *, const int *, const double *, const double *, const int *, - const double *, const int *, const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(cgemv)(const char *, const int *, const int *, const float *, const float *, const int *, - const float *, const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zgemv)(const char *, const int *, const int *, const double *, const double *, const int *, - const double *, const int *, const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xgemv)(const char *, const int *, const int *, const double *, const double *, const int *, - const double *, const int *, const double *, double *, const int *); +EIGEN_BLAS_API void BLASFUNC(sgemv)(const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, const float *, + const float *, const EIGEN_BLAS_INT *, const float *, const EIGEN_BLAS_INT *, + const float *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dgemv)(const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, const double *, + const double *, const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, + const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qgemv)(const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, const double *, + const double *, const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, + const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cgemv)(const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, const float *, + const float *, const EIGEN_BLAS_INT *, const float *, const EIGEN_BLAS_INT *, + const float *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zgemv)(const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, const double *, + const double *, const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, + const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xgemv)(const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, const double *, + const double *, const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, + const double *, double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(strsv)(const char *, const char *, const char *, const int *, const float *, const int *, - float *, const int *); -EIGEN_BLAS_API void BLASFUNC(dtrsv)(const char *, const char *, const char *, const int *, const double *, const int *, - double *, const int *); -EIGEN_BLAS_API void BLASFUNC(qtrsv)(const char *, const char *, const char *, const int *, const double *, const int *, - double *, const int *); -EIGEN_BLAS_API void BLASFUNC(ctrsv)(const char *, const char *, const char *, const int *, const float *, const int *, - float *, const int *); -EIGEN_BLAS_API void BLASFUNC(ztrsv)(const char *, const char *, const char *, const int *, const double *, const int *, - double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xtrsv)(const char *, const char *, const char *, const int *, const double *, const int *, - double *, const int *); +EIGEN_BLAS_API void BLASFUNC(strsv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const float *, + const EIGEN_BLAS_INT *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dtrsv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qtrsv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ctrsv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const float *, + const EIGEN_BLAS_INT *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ztrsv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xtrsv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(stpsv)(char *, char *, char *, int *, float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dtpsv)(char *, char *, char *, int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(qtpsv)(char *, char *, char *, int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(ctpsv)(char *, char *, char *, int *, float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(ztpsv)(char *, char *, char *, int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xtpsv)(char *, char *, char *, int *, double *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(stpsv)(char *, char *, char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dtpsv)(char *, char *, char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qtpsv)(char *, char *, char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ctpsv)(char *, char *, char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ztpsv)(char *, char *, char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xtpsv)(char *, char *, char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(strmv)(const char *, const char *, const char *, const int *, const float *, const int *, - float *, const int *); -EIGEN_BLAS_API void BLASFUNC(dtrmv)(const char *, const char *, const char *, const int *, const double *, const int *, - double *, const int *); -EIGEN_BLAS_API void BLASFUNC(qtrmv)(const char *, const char *, const char *, const int *, const double *, const int *, - double *, const int *); -EIGEN_BLAS_API void BLASFUNC(ctrmv)(const char *, const char *, const char *, const int *, const float *, const int *, - float *, const int *); -EIGEN_BLAS_API void BLASFUNC(ztrmv)(const char *, const char *, const char *, const int *, const double *, const int *, - double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xtrmv)(const char *, const char *, const char *, const int *, const double *, const int *, - double *, const int *); +EIGEN_BLAS_API void BLASFUNC(strmv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const float *, + const EIGEN_BLAS_INT *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dtrmv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qtrmv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ctrmv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const float *, + const EIGEN_BLAS_INT *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ztrmv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xtrmv)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(stpmv)(char *, char *, char *, int *, float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dtpmv)(char *, char *, char *, int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(qtpmv)(char *, char *, char *, int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(ctpmv)(char *, char *, char *, int *, float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(ztpmv)(char *, char *, char *, int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xtpmv)(char *, char *, char *, int *, double *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(stpmv)(char *, char *, char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dtpmv)(char *, char *, char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qtpmv)(char *, char *, char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ctpmv)(char *, char *, char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ztpmv)(char *, char *, char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xtpmv)(char *, char *, char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(stbmv)(char *, char *, char *, int *, int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dtbmv)(char *, char *, char *, int *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(qtbmv)(char *, char *, char *, int *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(ctbmv)(char *, char *, char *, int *, int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(ztbmv)(char *, char *, char *, int *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xtbmv)(char *, char *, char *, int *, int *, double *, int *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(stbmv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dtbmv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qtbmv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ctbmv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ztbmv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xtbmv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(stbsv)(char *, char *, char *, int *, int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dtbsv)(char *, char *, char *, int *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(qtbsv)(char *, char *, char *, int *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(ctbsv)(char *, char *, char *, int *, int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(ztbsv)(char *, char *, char *, int *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xtbsv)(char *, char *, char *, int *, int *, double *, int *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(stbsv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dtbsv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qtbsv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ctbsv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ztbsv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xtbsv)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(ssymv)(const char *, const int *, const float *, const float *, const int *, const float *, - const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(dsymv)(const char *, const int *, const double *, const double *, const int *, - const double *, const int *, const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(qsymv)(const char *, const int *, const double *, const double *, const int *, - const double *, const int *, const double *, double *, const int *); +EIGEN_BLAS_API void BLASFUNC(ssymv)(const char *, const EIGEN_BLAS_INT *, const float *, const float *, + const EIGEN_BLAS_INT *, const float *, const EIGEN_BLAS_INT *, const float *, + float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dsymv)(const char *, const EIGEN_BLAS_INT *, const double *, const double *, + const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, const double *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qsymv)(const char *, const EIGEN_BLAS_INT *, const double *, const double *, + const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, const double *, + double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(sspmv)(char *, int *, float *, float *, float *, int *, float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dspmv)(char *, int *, double *, double *, double *, int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(qspmv)(char *, int *, double *, double *, double *, int *, double *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(sspmv)(char *, EIGEN_BLAS_INT *, float *, float *, float *, EIGEN_BLAS_INT *, float *, + float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dspmv)(char *, EIGEN_BLAS_INT *, double *, double *, double *, EIGEN_BLAS_INT *, double *, + double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qspmv)(char *, EIGEN_BLAS_INT *, double *, double *, double *, EIGEN_BLAS_INT *, double *, + double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(ssyr)(const char *, const int *, const float *, const float *, const int *, float *, - const int *); -EIGEN_BLAS_API void BLASFUNC(dsyr)(const char *, const int *, const double *, const double *, const int *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(qsyr)(const char *, const int *, const double *, const double *, const int *, double *, - const int *); +EIGEN_BLAS_API void BLASFUNC(ssyr)(const char *, const EIGEN_BLAS_INT *, const float *, const float *, + const EIGEN_BLAS_INT *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dsyr)(const char *, const EIGEN_BLAS_INT *, const double *, const double *, + const EIGEN_BLAS_INT *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qsyr)(const char *, const EIGEN_BLAS_INT *, const double *, const double *, + const EIGEN_BLAS_INT *, double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(ssyr2)(const char *, const int *, const float *, const float *, const int *, const float *, - const int *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(dsyr2)(const char *, const int *, const double *, const double *, const int *, - const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(qsyr2)(const char *, const int *, const double *, const double *, const int *, - const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(csyr2)(const char *, const int *, const float *, const float *, const int *, const float *, - const int *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zsyr2)(const char *, const int *, const double *, const double *, const int *, - const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xsyr2)(const char *, const int *, const double *, const double *, const int *, - const double *, const int *, double *, const int *); +EIGEN_BLAS_API void BLASFUNC(ssyr2)(const char *, const EIGEN_BLAS_INT *, const float *, const float *, + const EIGEN_BLAS_INT *, const float *, const EIGEN_BLAS_INT *, float *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dsyr2)(const char *, const EIGEN_BLAS_INT *, const double *, const double *, + const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qsyr2)(const char *, const EIGEN_BLAS_INT *, const double *, const double *, + const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(csyr2)(const char *, const EIGEN_BLAS_INT *, const float *, const float *, + const EIGEN_BLAS_INT *, const float *, const EIGEN_BLAS_INT *, float *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zsyr2)(const char *, const EIGEN_BLAS_INT *, const double *, const double *, + const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xsyr2)(const char *, const EIGEN_BLAS_INT *, const double *, const double *, + const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, double *, + const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(sspr)(char *, int *, float *, float *, int *, float *); -EIGEN_BLAS_API void BLASFUNC(dspr)(char *, int *, double *, double *, int *, double *); -EIGEN_BLAS_API void BLASFUNC(qspr)(char *, int *, double *, double *, int *, double *); +EIGEN_BLAS_API void BLASFUNC(sspr)(char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *); +EIGEN_BLAS_API void BLASFUNC(dspr)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *); +EIGEN_BLAS_API void BLASFUNC(qspr)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *); -EIGEN_BLAS_API void BLASFUNC(sspr2)(char *, int *, float *, float *, int *, float *, int *, float *); -EIGEN_BLAS_API void BLASFUNC(dspr2)(char *, int *, double *, double *, int *, double *, int *, double *); -EIGEN_BLAS_API void BLASFUNC(qspr2)(char *, int *, double *, double *, int *, double *, int *, double *); -EIGEN_BLAS_API void BLASFUNC(cspr2)(char *, int *, float *, float *, int *, float *, int *, float *); -EIGEN_BLAS_API void BLASFUNC(zspr2)(char *, int *, double *, double *, int *, double *, int *, double *); -EIGEN_BLAS_API void BLASFUNC(xspr2)(char *, int *, double *, double *, int *, double *, int *, double *); +EIGEN_BLAS_API void BLASFUNC(sspr2)(char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *); +EIGEN_BLAS_API void BLASFUNC(dspr2)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *); +EIGEN_BLAS_API void BLASFUNC(qspr2)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *); +EIGEN_BLAS_API void BLASFUNC(cspr2)(char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *); +EIGEN_BLAS_API void BLASFUNC(zspr2)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *); +EIGEN_BLAS_API void BLASFUNC(xspr2)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *); -EIGEN_BLAS_API void BLASFUNC(cher)(char *, int *, float *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zher)(char *, int *, double *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xher)(char *, int *, double *, double *, int *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(cher)(char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zher)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xher)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(chpr)(char *, int *, float *, float *, int *, float *); -EIGEN_BLAS_API void BLASFUNC(zhpr)(char *, int *, double *, double *, int *, double *); -EIGEN_BLAS_API void BLASFUNC(xhpr)(char *, int *, double *, double *, int *, double *); +EIGEN_BLAS_API void BLASFUNC(chpr)(char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *); +EIGEN_BLAS_API void BLASFUNC(zhpr)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *); +EIGEN_BLAS_API void BLASFUNC(xhpr)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *); -EIGEN_BLAS_API void BLASFUNC(cher2)(char *, int *, float *, float *, int *, float *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zher2)(char *, int *, double *, double *, int *, double *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xher2)(char *, int *, double *, double *, int *, double *, int *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(cher2)(char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zher2)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xher2)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(chpr2)(char *, int *, float *, float *, int *, float *, int *, float *); -EIGEN_BLAS_API void BLASFUNC(zhpr2)(char *, int *, double *, double *, int *, double *, int *, double *); -EIGEN_BLAS_API void BLASFUNC(xhpr2)(char *, int *, double *, double *, int *, double *, int *, double *); +EIGEN_BLAS_API void BLASFUNC(chpr2)(char *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, float *, + EIGEN_BLAS_INT *, float *); +EIGEN_BLAS_API void BLASFUNC(zhpr2)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *); +EIGEN_BLAS_API void BLASFUNC(xhpr2)(char *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, double *, + EIGEN_BLAS_INT *, double *); -EIGEN_BLAS_API void BLASFUNC(chemv)(const char *, const int *, const float *, const float *, const int *, const float *, - const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zhemv)(const char *, const int *, const double *, const double *, const int *, - const double *, const int *, const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xhemv)(const char *, const int *, const double *, const double *, const int *, - const double *, const int *, const double *, double *, const int *); +EIGEN_BLAS_API void BLASFUNC(chemv)(const char *, const EIGEN_BLAS_INT *, const float *, const float *, + const EIGEN_BLAS_INT *, const float *, const EIGEN_BLAS_INT *, const float *, + float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zhemv)(const char *, const EIGEN_BLAS_INT *, const double *, const double *, + const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, const double *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xhemv)(const char *, const EIGEN_BLAS_INT *, const double *, const double *, + const EIGEN_BLAS_INT *, const double *, const EIGEN_BLAS_INT *, const double *, + double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(chpmv)(char *, int *, float *, float *, float *, int *, float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zhpmv)(char *, int *, double *, double *, double *, int *, double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xhpmv)(char *, int *, double *, double *, double *, int *, double *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(chpmv)(char *, EIGEN_BLAS_INT *, float *, float *, float *, EIGEN_BLAS_INT *, float *, + float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zhpmv)(char *, EIGEN_BLAS_INT *, double *, double *, double *, EIGEN_BLAS_INT *, double *, + double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xhpmv)(char *, EIGEN_BLAS_INT *, double *, double *, double *, EIGEN_BLAS_INT *, double *, + double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(snorm)(char *, int *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dnorm)(char *, int *, int *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(cnorm)(char *, int *, int *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(znorm)(char *, int *, int *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(snorm)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dnorm)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cnorm)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(znorm)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(sgbmv)(char *, int *, int *, int *, int *, float *, float *, int *, float *, int *, - float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dgbmv)(char *, int *, int *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(qgbmv)(char *, int *, int *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(cgbmv)(char *, int *, int *, int *, int *, float *, float *, int *, float *, int *, - float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zgbmv)(char *, int *, int *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xgbmv)(char *, int *, int *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(sgbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, + float *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, float *, + EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dgbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, + double *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, + double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qgbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, + double *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, + double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cgbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, + float *, float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, float *, + EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zgbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, + double *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, + double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xgbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, + double *, double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, + double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(ssbmv)(char *, int *, int *, float *, float *, int *, float *, int *, float *, float *, - int *); -EIGEN_BLAS_API void BLASFUNC(dsbmv)(char *, int *, int *, double *, double *, int *, double *, int *, double *, - double *, int *); -EIGEN_BLAS_API void BLASFUNC(qsbmv)(char *, int *, int *, double *, double *, int *, double *, int *, double *, - double *, int *); -EIGEN_BLAS_API void BLASFUNC(csbmv)(char *, int *, int *, float *, float *, int *, float *, int *, float *, float *, - int *); -EIGEN_BLAS_API void BLASFUNC(zsbmv)(char *, int *, int *, double *, double *, int *, double *, int *, double *, - double *, int *); -EIGEN_BLAS_API void BLASFUNC(xsbmv)(char *, int *, int *, double *, double *, int *, double *, int *, double *, - double *, int *); +EIGEN_BLAS_API void BLASFUNC(ssbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, + float *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dsbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, + double *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qsbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, + double *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(csbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, + float *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zsbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, + double *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xsbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, + double *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(chbmv)(char *, int *, int *, float *, float *, int *, float *, int *, float *, float *, - int *); -EIGEN_BLAS_API void BLASFUNC(zhbmv)(char *, int *, int *, double *, double *, int *, double *, int *, double *, - double *, int *); -EIGEN_BLAS_API void BLASFUNC(xhbmv)(char *, int *, int *, double *, double *, int *, double *, int *, double *, - double *, int *); +EIGEN_BLAS_API void BLASFUNC(chbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *, + float *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zhbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, + double *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xhbmv)(char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *, + double *, EIGEN_BLAS_INT *, double *, double *, EIGEN_BLAS_INT *); /* Level 3 routines */ -EIGEN_BLAS_API void BLASFUNC(sgemm)(const char *, const char *, const int *, const int *, const int *, const float *, - const float *, const int *, const float *, const int *, const float *, float *, - const int *); -EIGEN_BLAS_API void BLASFUNC(dgemm)(const char *, const char *, const int *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(qgemm)(const char *, const char *, const int *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(cgemm)(const char *, const char *, const int *, const int *, const int *, const float *, - const float *, const int *, const float *, const int *, const float *, float *, - const int *); -EIGEN_BLAS_API void BLASFUNC(zgemm)(const char *, const char *, const int *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(xgemm)(const char *, const char *, const int *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); +EIGEN_BLAS_API void BLASFUNC(sgemm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + const float *, const EIGEN_BLAS_INT *, const float *, float *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dgemm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qgemm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cgemm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + const float *, const EIGEN_BLAS_INT *, const float *, float *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zgemm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xgemm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(cgemm3m)(char *, char *, int *, int *, int *, float *, float *, int *, float *, int *, - float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zgemm3m)(char *, char *, int *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xgemm3m)(char *, char *, int *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(cgemm3m)(char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, + float *, EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, float *, + EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zgemm3m)(char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, + double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xgemm3m)(char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, + double *, EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(sge2mm)(char *, char *, char *, int *, int *, float *, float *, int *, float *, int *, - float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(dge2mm)(char *, char *, char *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(cge2mm)(char *, char *, char *, int *, int *, float *, float *, int *, float *, int *, - float *, float *, int *); -EIGEN_BLAS_API void BLASFUNC(zge2mm)(char *, char *, char *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(sge2mm)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dge2mm)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cge2mm)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zge2mm)(char *, char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(strsm)(const char *, const char *, const char *, const char *, const int *, const int *, - const float *, const float *, const int *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(dtrsm)(const char *, const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(qtrsm)(const char *, const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(ctrsm)(const char *, const char *, const char *, const char *, const int *, const int *, - const float *, const float *, const int *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(ztrsm)(const char *, const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xtrsm)(const char *, const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, double *, const int *); +EIGEN_BLAS_API void BLASFUNC(strsm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dtrsm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qtrsm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ctrsm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ztrsm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xtrsm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(strmm)(const char *, const char *, const char *, const char *, const int *, const int *, - const float *, const float *, const int *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(dtrmm)(const char *, const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(qtrmm)(const char *, const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(ctrmm)(const char *, const char *, const char *, const char *, const int *, const int *, - const float *, const float *, const int *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(ztrmm)(const char *, const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xtrmm)(const char *, const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, double *, const int *); +EIGEN_BLAS_API void BLASFUNC(strmm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dtrmm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qtrmm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ctrmm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(ztrmm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xtrmm)(const char *, const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(ssymm)(const char *, const char *, const int *, const int *, const float *, const float *, - const int *, const float *, const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(dsymm)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(qsymm)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(csymm)(const char *, const char *, const int *, const int *, const float *, const float *, - const int *, const float *, const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zsymm)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(xsymm)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); +EIGEN_BLAS_API void BLASFUNC(ssymm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const float *, const float *, const EIGEN_BLAS_INT *, const float *, + const EIGEN_BLAS_INT *, const float *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dsymm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qsymm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(csymm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const float *, const float *, const EIGEN_BLAS_INT *, const float *, + const EIGEN_BLAS_INT *, const float *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zsymm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xsymm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(csymm3m)(char *, char *, int *, int *, float *, float *, int *, float *, int *, float *, - float *, int *); -EIGEN_BLAS_API void BLASFUNC(zsymm3m)(char *, char *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xsymm3m)(char *, char *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(csymm3m)(char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zsymm3m)(char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xsymm3m)(char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(ssyrk)(const char *, const char *, const int *, const int *, const float *, const float *, - const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(dsyrk)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(qsyrk)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(csyrk)(const char *, const char *, const int *, const int *, const float *, const float *, - const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zsyrk)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xsyrk)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, double *, const int *); +EIGEN_BLAS_API void BLASFUNC(ssyrk)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const float *, const float *, const EIGEN_BLAS_INT *, const float *, float *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dsyrk)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qsyrk)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(csyrk)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const float *, const float *, const EIGEN_BLAS_INT *, const float *, float *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zsyrk)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xsyrk)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(ssyr2k)(const char *, const char *, const int *, const int *, const float *, const float *, - const int *, const float *, const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(dsyr2k)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(qsyr2k)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(csyr2k)(const char *, const char *, const int *, const int *, const float *, const float *, - const int *, const float *, const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zsyr2k)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(xsyr2k)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); +EIGEN_BLAS_API void BLASFUNC(ssyr2k)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const float *, const float *, const EIGEN_BLAS_INT *, const float *, + const EIGEN_BLAS_INT *, const float *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dsyr2k)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qsyr2k)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(csyr2k)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const float *, const float *, const EIGEN_BLAS_INT *, const float *, + const EIGEN_BLAS_INT *, const float *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zsyr2k)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xsyr2k)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(chemm)(const char *, const char *, const int *, const int *, const float *, const float *, - const int *, const float *, const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zhemm)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(xhemm)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); +EIGEN_BLAS_API void BLASFUNC(chemm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const float *, const float *, const EIGEN_BLAS_INT *, const float *, + const EIGEN_BLAS_INT *, const float *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zhemm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xhemm)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(chemm3m)(char *, char *, int *, int *, float *, float *, int *, float *, int *, float *, - float *, int *); -EIGEN_BLAS_API void BLASFUNC(zhemm3m)(char *, char *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); -EIGEN_BLAS_API void BLASFUNC(xhemm3m)(char *, char *, int *, int *, double *, double *, int *, double *, int *, - double *, double *, int *); +EIGEN_BLAS_API void BLASFUNC(chemm3m)(char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, float *, float *, + EIGEN_BLAS_INT *, float *, EIGEN_BLAS_INT *, float *, float *, EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zhemm3m)(char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xhemm3m)(char *, char *, EIGEN_BLAS_INT *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *, double *, EIGEN_BLAS_INT *, double *, double *, + EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(cherk)(const char *, const char *, const int *, const int *, const float *, const float *, - const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zherk)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xherk)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, double *, const int *); +EIGEN_BLAS_API void BLASFUNC(cherk)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const float *, const float *, const EIGEN_BLAS_INT *, const float *, float *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zherk)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xherk)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(cher2k)(const char *, const char *, const int *, const int *, const float *, const float *, - const int *, const float *, const int *, const float *, float *, const int *); -EIGEN_BLAS_API void BLASFUNC(zher2k)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(xher2k)(const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(cher2m)(const char *, const char *, const char *, const int *, const int *, const float *, - const float *, const int *, const float *, const int *, const float *, float *, - const int *); -EIGEN_BLAS_API void BLASFUNC(zher2m)(const char *, const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); -EIGEN_BLAS_API void BLASFUNC(xher2m)(const char *, const char *, const char *, const int *, const int *, const double *, - const double *, const int *, const double *, const int *, const double *, double *, - const int *); +EIGEN_BLAS_API void BLASFUNC(cher2k)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const float *, const float *, const EIGEN_BLAS_INT *, const float *, + const EIGEN_BLAS_INT *, const float *, float *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zher2k)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xher2k)(const char *, const char *, const EIGEN_BLAS_INT *, const EIGEN_BLAS_INT *, + const double *, const double *, const EIGEN_BLAS_INT *, const double *, + const EIGEN_BLAS_INT *, const double *, double *, const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cher2m)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + const float *, const EIGEN_BLAS_INT *, const float *, float *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zher2m)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xher2m)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); -EIGEN_BLAS_API void BLASFUNC(sgemmtr)(const char *, const char *, const char *, const int *, const int *, const float *, - const float *, const int *, const float *, const int *, const float *, float *, - const int *); -EIGEN_BLAS_API void BLASFUNC(dgemmtr)(const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, const double *, const int *, - const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(qgemmtr)(const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, const double *, const int *, - const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(cgemmtr)(const char *, const char *, const char *, const int *, const int *, const float *, - const float *, const int *, const float *, const int *, const float *, float *, - const int *); -EIGEN_BLAS_API void BLASFUNC(zgemmtr)(const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, const double *, const int *, - const double *, double *, const int *); -EIGEN_BLAS_API void BLASFUNC(xgemmtr)(const char *, const char *, const char *, const int *, const int *, - const double *, const double *, const int *, const double *, const int *, - const double *, double *, const int *); +EIGEN_BLAS_API void BLASFUNC(sgemmtr)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + const float *, const EIGEN_BLAS_INT *, const float *, float *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(dgemmtr)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(qgemmtr)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(cgemmtr)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const float *, const float *, const EIGEN_BLAS_INT *, + const float *, const EIGEN_BLAS_INT *, const float *, float *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(zgemmtr)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); +EIGEN_BLAS_API void BLASFUNC(xgemmtr)(const char *, const char *, const char *, const EIGEN_BLAS_INT *, + const EIGEN_BLAS_INT *, const double *, const double *, const EIGEN_BLAS_INT *, + const double *, const EIGEN_BLAS_INT *, const double *, double *, + const EIGEN_BLAS_INT *); #ifdef __cplusplus }
diff --git a/blas/common.h b/blas/common.h index 5779185..f4bf358 100644 --- a/blas/common.h +++ b/blas/common.h
@@ -88,46 +88,45 @@ typedef Eigen::Map<Eigen::Matrix<Scalar, Eigen::Dynamic, 1> > CompactVectorType; template <typename T> -Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>, 0, Eigen::OuterStride<> > matrix( - T* data, int rows, int cols, int stride) { +Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>, 0, Eigen::OuterStride<>> matrix( + T* data, EIGEN_BLAS_INT rows, EIGEN_BLAS_INT cols, EIGEN_BLAS_INT stride) { return Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>, 0, Eigen::OuterStride<> >( data, rows, cols, Eigen::OuterStride<>(stride)); } template <typename T> -Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>, 0, Eigen::OuterStride<> > matrix( - const T* data, int rows, int cols, int stride) { +Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>, 0, Eigen::OuterStride<>> matrix( + const T* data, EIGEN_BLAS_INT rows, EIGEN_BLAS_INT cols, EIGEN_BLAS_INT stride) { return Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>, 0, Eigen::OuterStride<> >( data, rows, cols, Eigen::OuterStride<>(stride)); } template <typename T> -Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, 1>, 0, Eigen::InnerStride<Eigen::Dynamic> > make_vector(T* data, int size, - int incr) { +Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, 1>, 0, Eigen::InnerStride<Eigen::Dynamic>> make_vector( + T* data, EIGEN_BLAS_INT size, EIGEN_BLAS_INT incr) { return Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, 1>, 0, Eigen::InnerStride<Eigen::Dynamic> >( data, size, Eigen::InnerStride<Eigen::Dynamic>(incr)); } template <typename T> -Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1>, 0, Eigen::InnerStride<Eigen::Dynamic> > make_vector(const T* data, - int size, - int incr) { +Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1>, 0, Eigen::InnerStride<Eigen::Dynamic>> make_vector( + const T* data, EIGEN_BLAS_INT size, EIGEN_BLAS_INT incr) { return Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1>, 0, Eigen::InnerStride<Eigen::Dynamic> >( data, size, Eigen::InnerStride<Eigen::Dynamic>(incr)); } template <typename T> -Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, 1> > make_vector(T* data, int size) { +Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, 1>> make_vector(T* data, EIGEN_BLAS_INT size) { return Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, 1> >(data, size); } template <typename T> -Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1> > make_vector(const T* data, int size) { +Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1>> make_vector(const T* data, EIGEN_BLAS_INT size) { return Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1> >(data, size); } template <typename T> -T* get_compact_vector(T* x, int n, int incx) { +T* get_compact_vector(T* x, EIGEN_BLAS_INT n, EIGEN_BLAS_INT incx) { if (incx == 1) return x; std::remove_const_t<T>* ret = new Scalar[n]; @@ -139,7 +138,7 @@ } template <typename T> -T* copy_back(T* x_cpy, T* x, int n, int incx) { +T* copy_back(T* x_cpy, T* x, EIGEN_BLAS_INT n, EIGEN_BLAS_INT incx) { if (x_cpy == x) return 0; if (incx < 0)
diff --git a/blas/complexdots.cpp b/blas/complexdots.cpp index 253edbb..c0bbbbf 100644 --- a/blas/complexdots.cpp +++ b/blas/complexdots.cpp
@@ -27,13 +27,23 @@ #define EIGEN_BLAS_CDOT_API #endif +#include <stdint.h> + +#ifndef EIGEN_BLAS_INT +#if defined(EIGEN_64BIT_BLAS) +#define EIGEN_BLAS_INT int64_t +#else +#define EIGEN_BLAS_INT int +#endif +#endif + extern "C" { // Worker function declarations (defined in level1_cplx_impl.h via complex_single.cpp / complex_double.cpp). -void cdotcw_(int *n, float *cx, int *incx, float *cy, int *incy, float *res); -void cdotuw_(int *n, float *cx, int *incx, float *cy, int *incy, float *res); -void zdotcw_(int *n, double *cx, int *incx, double *cy, int *incy, double *res); -void zdotuw_(int *n, double *cx, int *incx, double *cy, int *incy, double *res); +void cdotcw_(EIGEN_BLAS_INT *n, float *cx, EIGEN_BLAS_INT *incx, float *cy, EIGEN_BLAS_INT *incy, float *res); +void cdotuw_(EIGEN_BLAS_INT *n, float *cx, EIGEN_BLAS_INT *incx, float *cy, EIGEN_BLAS_INT *incy, float *res); +void zdotcw_(EIGEN_BLAS_INT *n, double *cx, EIGEN_BLAS_INT *incx, double *cy, EIGEN_BLAS_INT *incy, double *res); +void zdotuw_(EIGEN_BLAS_INT *n, double *cx, EIGEN_BLAS_INT *incx, double *cy, EIGEN_BLAS_INT *incy, double *res); // POD complex types for C-compatible return values (matches Fortran complex layout). struct eigen_blas_complex_float { @@ -44,28 +54,32 @@ }; // CDOTC computes the conjugated dot product of two single-precision complex vectors. -EIGEN_BLAS_CDOT_API eigen_blas_complex_float cdotc_(int *n, float *cx, int *incx, float *cy, int *incy) { +EIGEN_BLAS_CDOT_API eigen_blas_complex_float cdotc_(EIGEN_BLAS_INT *n, float *cx, EIGEN_BLAS_INT *incx, float *cy, + EIGEN_BLAS_INT *incy) { eigen_blas_complex_float res = {0.0f, 0.0f}; cdotcw_(n, cx, incx, cy, incy, &res.r); return res; } // CDOTU computes the unconjugated dot product of two single-precision complex vectors. -EIGEN_BLAS_CDOT_API eigen_blas_complex_float cdotu_(int *n, float *cx, int *incx, float *cy, int *incy) { +EIGEN_BLAS_CDOT_API eigen_blas_complex_float cdotu_(EIGEN_BLAS_INT *n, float *cx, EIGEN_BLAS_INT *incx, float *cy, + EIGEN_BLAS_INT *incy) { eigen_blas_complex_float res = {0.0f, 0.0f}; cdotuw_(n, cx, incx, cy, incy, &res.r); return res; } // ZDOTC computes the conjugated dot product of two double-precision complex vectors. -EIGEN_BLAS_CDOT_API eigen_blas_complex_double zdotc_(int *n, double *cx, int *incx, double *cy, int *incy) { +EIGEN_BLAS_CDOT_API eigen_blas_complex_double zdotc_(EIGEN_BLAS_INT *n, double *cx, EIGEN_BLAS_INT *incx, double *cy, + EIGEN_BLAS_INT *incy) { eigen_blas_complex_double res = {0.0, 0.0}; zdotcw_(n, cx, incx, cy, incy, &res.r); return res; } // ZDOTU computes the unconjugated dot product of two double-precision complex vectors. -EIGEN_BLAS_CDOT_API eigen_blas_complex_double zdotu_(int *n, double *cx, int *incx, double *cy, int *incy) { +EIGEN_BLAS_CDOT_API eigen_blas_complex_double zdotu_(EIGEN_BLAS_INT *n, double *cx, EIGEN_BLAS_INT *incx, double *cy, + EIGEN_BLAS_INT *incy) { eigen_blas_complex_double res = {0.0, 0.0}; zdotuw_(n, cx, incx, cy, incy, &res.r); return res;
diff --git a/blas/double.cpp b/blas/double.cpp index eb99aae..ce5ed6e 100644 --- a/blas/double.cpp +++ b/blas/double.cpp
@@ -20,7 +20,8 @@ #include "level2_real_impl.h" #include "level3_impl.h" -extern "C" double EIGEN_BLAS_FUNC_NAME(sdot)(int* n, float* x, int* incx, float* y, int* incy) { +extern "C" double EIGEN_BLAS_FUNC_NAME(sdot)(EIGEN_BLAS_INT* n, float* x, EIGEN_BLAS_INT* incx, float* y, + EIGEN_BLAS_INT* incy) { if (*n <= 0) return 0; if (*incx == 1 && *incy == 1)
diff --git a/blas/eigen_blas.def b/blas/eigen_blas.def index 1589d51..2336c43 100644 --- a/blas/eigen_blas.def +++ b/blas/eigen_blas.def
@@ -1,6 +1,7 @@ -; Definition file for eigen_blas.dll. +; Definition file for eigen_blas.dll and eigen_blas_ilp64.dll. +; No LIBRARY statement: the linker names each DLL after its /OUT file, so the +; ILP64 import library does not resolve to eigen_blas.dll. -LIBRARY eigen_blas EXPORTS ; Utilities lsame_ @@ -13,10 +14,10 @@ zaxpy_ ; caxpyc_ ; zaxpyc_ - saxpby_, - daxpby_, - caxpby_, - zaxpby_, + saxpby_ + daxpby_ + caxpby_ + zaxpby_ scopy_ dcopy_ ccopy_
diff --git a/blas/level1_cplx_impl.h b/blas/level1_cplx_impl.h index 314d1a7..517f65e 100644 --- a/blas/level1_cplx_impl.h +++ b/blas/level1_cplx_impl.h
@@ -30,7 +30,8 @@ // computes the sum of magnitudes of all vector elements or, for a complex vector x, the sum // res = |Rex1| + |Imx1| + |Rex2| + |Imx2| + ... + |Rexn| + |Imxn|, where x is a vector of order n -extern "C" RealScalar EIGEN_CAT(REAL_SCALAR_SUFFIX, EIGEN_BLAS_FUNC_NAME(asum))(int *n, RealScalar *px, int *incx) { +extern "C" RealScalar EIGEN_CAT(REAL_SCALAR_SUFFIX, EIGEN_BLAS_FUNC_NAME(asum))(EIGEN_BLAS_INT *n, RealScalar *px, + EIGEN_BLAS_INT *incx) { if (*n <= 0) return 0; // std::complex<T> is layout-compatible with T[2], so we can reinterpret @@ -46,7 +47,8 @@ } } -extern "C" int EIGEN_CAT(i, EIGEN_BLAS_FUNC_NAME(amax))(int *n, RealScalar *px, int *incx) { +extern "C" EIGEN_BLAS_INT EIGEN_CAT(i, EIGEN_BLAS_FUNC_NAME(amax))(EIGEN_BLAS_INT *n, RealScalar *px, + EIGEN_BLAS_INT *incx) { if (*n <= 0) return 0; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -55,10 +57,11 @@ make_vector(x, *n).unaryExpr<scalar_norm1_op>().maxCoeff(&ret); else make_vector(x, *n, std::abs(*incx)).unaryExpr<scalar_norm1_op>().maxCoeff(&ret); - return int(ret) + 1; + return EIGEN_BLAS_INT(ret) + 1; } -extern "C" int EIGEN_CAT(i, EIGEN_BLAS_FUNC_NAME(amin))(int *n, RealScalar *px, int *incx) { +extern "C" EIGEN_BLAS_INT EIGEN_CAT(i, EIGEN_BLAS_FUNC_NAME(amin))(EIGEN_BLAS_INT *n, RealScalar *px, + EIGEN_BLAS_INT *incx) { if (*n <= 0) return 0; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -67,11 +70,12 @@ make_vector(x, *n).unaryExpr<scalar_norm1_op>().minCoeff(&ret); else make_vector(x, *n, std::abs(*incx)).unaryExpr<scalar_norm1_op>().minCoeff(&ret); - return int(ret) + 1; + return EIGEN_BLAS_INT(ret) + 1; } // computes a dot product of a conjugated vector with another vector. -EIGEN_BLAS_FUNC(dotcw)(int *n, RealScalar *px, int *incx, RealScalar *py, int *incy, RealScalar *pres) { +EIGEN_BLAS_FUNC(dotcw) +(EIGEN_BLAS_INT *n, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *py, EIGEN_BLAS_INT *incy, RealScalar *pres) { // std::cerr << "_dotc " << *n << " " << *incx << " " << *incy << "\n"; Scalar *res = reinterpret_cast<Scalar *>(pres); @@ -96,7 +100,8 @@ } // computes a vector-vector dot product without complex conjugation. -EIGEN_BLAS_FUNC(dotuw)(int *n, RealScalar *px, int *incx, RealScalar *py, int *incy, RealScalar *pres) { +EIGEN_BLAS_FUNC(dotuw) +(EIGEN_BLAS_INT *n, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *py, EIGEN_BLAS_INT *incy, RealScalar *pres) { Scalar *res = reinterpret_cast<Scalar *>(pres); if (*n <= 0) { @@ -119,7 +124,8 @@ *res = (make_vector(x, *n, -*incx).reverse().cwiseProduct(make_vector(y, *n, -*incy).reverse())).sum(); } -extern "C" RealScalar EIGEN_CAT(REAL_SCALAR_SUFFIX, EIGEN_BLAS_FUNC_NAME(nrm2))(int *n, RealScalar *px, int *incx) { +extern "C" RealScalar EIGEN_CAT(REAL_SCALAR_SUFFIX, EIGEN_BLAS_FUNC_NAME(nrm2))(EIGEN_BLAS_INT *n, RealScalar *px, + EIGEN_BLAS_INT *incx) { // std::cerr << "__nrm2 " << *n << " " << *incx << "\n"; if (*n <= 0) return 0; @@ -131,7 +137,8 @@ } EIGEN_BLAS_FUNC(EIGEN_CAT(REAL_SCALAR_SUFFIX, rot)) -(int *n, RealScalar *px, int *incx, RealScalar *py, int *incy, RealScalar *pc, RealScalar *ps) { +(EIGEN_BLAS_INT *n, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *py, EIGEN_BLAS_INT *incy, RealScalar *pc, + RealScalar *ps) { if (*n <= 0) return; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -154,7 +161,8 @@ Eigen::internal::apply_rotation_in_the_plane(vx, vy, Eigen::JacobiRotation<Scalar>(c, s)); } -EIGEN_BLAS_FUNC(EIGEN_CAT(REAL_SCALAR_SUFFIX, scal))(int *n, RealScalar *palpha, RealScalar *px, int *incx) { +EIGEN_BLAS_FUNC(EIGEN_CAT(REAL_SCALAR_SUFFIX, scal)) +(EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *px, EIGEN_BLAS_INT *incx) { if (*n <= 0) return; Scalar *x = reinterpret_cast<Scalar *>(px);
diff --git a/blas/level1_impl.h b/blas/level1_impl.h index 525a628..b62b864 100644 --- a/blas/level1_impl.h +++ b/blas/level1_impl.h
@@ -11,7 +11,8 @@ #include "common.h" EIGEN_BLAS_FUNC(axpy) -(const int *n, const RealScalar *palpha, const RealScalar *px, const int *incx, RealScalar *py, const int *incy) { +(const EIGEN_BLAS_INT *n, const RealScalar *palpha, const RealScalar *px, const EIGEN_BLAS_INT *incx, RealScalar *py, + const EIGEN_BLAS_INT *incy) { const Scalar *x = reinterpret_cast<const Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); @@ -31,13 +32,13 @@ } EIGEN_BLAS_FUNC(axpby) -(const int *pn, const RealScalar *palpha, const RealScalar *px, const int *pincx, const RealScalar *pbeta, - RealScalar *py, const int *pincy) { +(const EIGEN_BLAS_INT *pn, const RealScalar *palpha, const RealScalar *px, const EIGEN_BLAS_INT *pincx, + const RealScalar *pbeta, RealScalar *py, const EIGEN_BLAS_INT *pincy) { const Scalar *x = reinterpret_cast<const Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); const Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); const Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); - const int n = *pn; + const EIGEN_BLAS_INT n = *pn; if (n <= 0) return; @@ -46,8 +47,8 @@ return; } - const int incx = *pincx; - const int incy = *pincy; + const EIGEN_BLAS_INT incx = *pincx; + const EIGEN_BLAS_INT incy = *pincy; if (incx == 1 && incy == 1) make_vector(y, n) = alpha * make_vector(x, n) + beta * make_vector(y, n); @@ -61,7 +62,7 @@ make_vector(y, n, -incy) = alpha * make_vector(x, n, -incx) + beta * make_vector(y, n, -incy); } -EIGEN_BLAS_FUNC(copy)(int *n, RealScalar *px, int *incx, RealScalar *py, int *incy) { +EIGEN_BLAS_FUNC(copy)(EIGEN_BLAS_INT *n, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *py, EIGEN_BLAS_INT *incy) { if (*n <= 0) return; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -73,7 +74,7 @@ else if (*incx == 0) { // Broadcast: copy x[0] to all elements of y. if (*incy < 0) y = y - (*n - 1) * (*incy); - for (int i = 0; i < *n; ++i) { + for (EIGEN_BLAS_INT i = 0; i < *n; ++i) { *y = *x; y += *incy; } @@ -140,7 +141,7 @@ // *s = r.s(); } -EIGEN_BLAS_FUNC(scal)(int *n, RealScalar *palpha, RealScalar *px, int *incx) { +EIGEN_BLAS_FUNC(scal)(EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *px, EIGEN_BLAS_INT *incx) { if (*n <= 0) return; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -152,7 +153,7 @@ make_vector(x, *n, std::abs(*incx)) *= alpha; } -EIGEN_BLAS_FUNC(swap)(int *n, RealScalar *px, int *incx, RealScalar *py, int *incy) { +EIGEN_BLAS_FUNC(swap)(EIGEN_BLAS_INT *n, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *py, EIGEN_BLAS_INT *incy) { if (*n <= 0) return; Scalar *x = reinterpret_cast<Scalar *>(px);
diff --git a/blas/level1_real_impl.h b/blas/level1_real_impl.h index 7a213dd..55eaa5d 100644 --- a/blas/level1_real_impl.h +++ b/blas/level1_real_impl.h
@@ -12,7 +12,7 @@ // computes the sum of magnitudes of all vector elements or, for a complex vector x, the sum // res = |Rex1| + |Imx1| + |Rex2| + |Imx2| + ... + |Rexn| + |Imxn|, where x is a vector of order n -extern "C" RealScalar EIGEN_BLAS_FUNC_NAME(asum)(int *n, Scalar *px, int *incx) { +extern "C" RealScalar EIGEN_BLAS_FUNC_NAME(asum)(EIGEN_BLAS_INT *n, Scalar *px, EIGEN_BLAS_INT *incx) { // std::cerr << "_asum " << *n << " " << *incx << "\n"; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -25,7 +25,8 @@ return make_vector(x, *n, std::abs(*incx)).cwiseAbs().sum(); } -extern "C" int EIGEN_CAT(i, EIGEN_BLAS_FUNC_NAME(amax))(int *n, Scalar *px, int *incx) { +extern "C" EIGEN_BLAS_INT EIGEN_CAT(i, EIGEN_BLAS_FUNC_NAME(amax))(EIGEN_BLAS_INT *n, Scalar *px, + EIGEN_BLAS_INT *incx) { if (*n <= 0) return 0; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -34,10 +35,11 @@ make_vector(x, *n).cwiseAbs().maxCoeff(&ret); else make_vector(x, *n, std::abs(*incx)).cwiseAbs().maxCoeff(&ret); - return int(ret) + 1; + return EIGEN_BLAS_INT(ret) + 1; } -extern "C" int EIGEN_CAT(i, EIGEN_BLAS_FUNC_NAME(amin))(int *n, Scalar *px, int *incx) { +extern "C" EIGEN_BLAS_INT EIGEN_CAT(i, EIGEN_BLAS_FUNC_NAME(amin))(EIGEN_BLAS_INT *n, Scalar *px, + EIGEN_BLAS_INT *incx) { if (*n <= 0) return 0; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -46,11 +48,12 @@ make_vector(x, *n).cwiseAbs().minCoeff(&ret); else make_vector(x, *n, std::abs(*incx)).cwiseAbs().minCoeff(&ret); - return int(ret) + 1; + return EIGEN_BLAS_INT(ret) + 1; } // computes a vector-vector dot product. -extern "C" Scalar EIGEN_BLAS_FUNC_NAME(dot)(int *n, Scalar *px, int *incx, Scalar *py, int *incy) { +extern "C" Scalar EIGEN_BLAS_FUNC_NAME(dot)(EIGEN_BLAS_INT *n, Scalar *px, EIGEN_BLAS_INT *incx, Scalar *py, + EIGEN_BLAS_INT *incy) { // std::cerr << "_dot " << *n << " " << *incx << " " << *incy << "\n"; if (*n <= 0) return 0; @@ -73,7 +76,7 @@ } // computes the Euclidean norm of a vector. -extern "C" Scalar EIGEN_BLAS_FUNC_NAME(nrm2)(int *n, Scalar *px, int *incx) { +extern "C" Scalar EIGEN_BLAS_FUNC_NAME(nrm2)(EIGEN_BLAS_INT *n, Scalar *px, EIGEN_BLAS_INT *incx) { if (*n <= 0) return 0; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -84,7 +87,8 @@ return make_vector(x, *n, std::abs(*incx)).stableNorm(); } -EIGEN_BLAS_FUNC(rot)(int *n, Scalar *px, int *incx, Scalar *py, int *incy, Scalar *pc, Scalar *ps) { +EIGEN_BLAS_FUNC(rot) +(EIGEN_BLAS_INT *n, Scalar *px, EIGEN_BLAS_INT *incx, Scalar *py, EIGEN_BLAS_INT *incy, Scalar *pc, Scalar *ps) { // std::cerr << "_rot " << *n << " " << *incx << " " << *incy << "\n"; if (*n <= 0) return; @@ -114,7 +118,8 @@ // 1: H = [[h11, 1], [-1, h22]] (h11, h22 from param) // -2: H = identity (no-op) // param[1..4] = h11, h21, h12, h22 -EIGEN_BLAS_FUNC(rotm)(int *n, Scalar *px, int *incx, Scalar *py, int *incy, Scalar *param) { +EIGEN_BLAS_FUNC(rotm) +(EIGEN_BLAS_INT *n, Scalar *px, EIGEN_BLAS_INT *incx, Scalar *py, EIGEN_BLAS_INT *incy, Scalar *param) { Scalar *x = reinterpret_cast<Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); @@ -139,10 +144,10 @@ h22 = param[4]; } - int kx = *incx > 0 ? 0 : (1 - *n) * *incx; - int ky = *incy > 0 ? 0 : (1 - *n) * *incy; + EIGEN_BLAS_INT kx = *incx > 0 ? 0 : (1 - *n) * *incx; + EIGEN_BLAS_INT ky = *incy > 0 ? 0 : (1 - *n) * *incy; - for (int i = 0; i < *n; ++i) { + for (EIGEN_BLAS_INT i = 0; i < *n; ++i) { Scalar w = x[kx]; Scalar z = y[ky]; x[kx] = h11 * w + h12 * z;
diff --git a/blas/level2_cplx_impl.h b/blas/level2_cplx_impl.h index 60d6869..9c69fb0 100644 --- a/blas/level2_cplx_impl.h +++ b/blas/level2_cplx_impl.h
@@ -18,15 +18,16 @@ * A is an n by n hermitian matrix. */ EIGEN_BLAS_FUNC(hemv) -(const char *uplo, const int *n, const RealScalar *palpha, const RealScalar *pa, const int *lda, const RealScalar *px, - const int *incx, const RealScalar *pbeta, RealScalar *py, const int *incy) { - typedef void (*functype)(int, const Scalar *, int, const Scalar *, Scalar *, Scalar); +(const char *uplo, const EIGEN_BLAS_INT *n, const RealScalar *palpha, const RealScalar *pa, const EIGEN_BLAS_INT *lda, + const RealScalar *px, const EIGEN_BLAS_INT *incx, const RealScalar *pbeta, RealScalar *py, + const EIGEN_BLAS_INT *incy) { + typedef void (*functype)(EIGEN_BLAS_INT, const Scalar *, EIGEN_BLAS_INT, const Scalar *, Scalar *, Scalar); static const functype func[2] = { // array index: UP - (Eigen::internal::selfadjoint_matrix_vector_product<Scalar, int, Eigen::ColMajor, Eigen::Upper, false, + (Eigen::internal::selfadjoint_matrix_vector_product<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, Eigen::Upper, false, false>::run), // array index: LO - (Eigen::internal::selfadjoint_matrix_vector_product<Scalar, int, Eigen::ColMajor, Eigen::Lower, false, + (Eigen::internal::selfadjoint_matrix_vector_product<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, Eigen::Lower, false, false>::run), }; @@ -37,12 +38,12 @@ Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); // check arguments - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) info = 2; - else if (*lda < std::max(1, *n)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, *n)) info = 5; else if (*incx == 0) info = 7; @@ -82,15 +83,15 @@ * Diagonal elements are real; off-diagonal contributions use conjugation. */ EIGEN_BLAS_FUNC(hbmv) -(char *uplo, int *n, int *k, RealScalar *palpha, RealScalar *pa, int *lda, RealScalar *px, int *incx, RealScalar *pbeta, - RealScalar *py, int *incy) { +(char *uplo, EIGEN_BLAS_INT *n, EIGEN_BLAS_INT *k, RealScalar *palpha, RealScalar *pa, EIGEN_BLAS_INT *lda, + RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *pbeta, RealScalar *py, EIGEN_BLAS_INT *incy) { const Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); const Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); const Scalar *a = reinterpret_cast<const Scalar *>(pa); const Scalar *x = reinterpret_cast<const Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) @@ -128,10 +129,10 @@ // Vectorized path: use Eigen Map segments for the inner band operations. ConstMatrixType band(a, *k + 1, *n, *lda); if (UPLO(*uplo) == UP) { - for (int j = 0; j < *n; ++j) { - int start = std::max(0, j - *k); - int len = j - start; - int offset = *k - (j - start); + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { + EIGEN_BLAS_INT start = std::max<EIGEN_BLAS_INT>(0, j - *k); + EIGEN_BLAS_INT len = j - start; + EIGEN_BLAS_INT offset = *k - (j - start); Scalar temp1 = alpha * actual_x[j]; actual_y[j] += Scalar(Eigen::numext::real(band(*k, j))) * temp1; if (len > 0) { @@ -140,8 +141,8 @@ } } } else { - for (int j = 0; j < *n; ++j) { - int len = std::min(*n - 1, j + *k) - j; + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { + EIGEN_BLAS_INT len = std::min(*n - 1, j + *k) - j; Scalar temp1 = alpha * actual_x[j]; actual_y[j] += Scalar(Eigen::numext::real(band(0, j))) * temp1; if (len > 0) { @@ -153,10 +154,10 @@ } else { // Scalar path: for narrow bandwidth, avoid Map overhead. if (UPLO(*uplo) == UP) { - for (int j = 0; j < *n; ++j) { + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { Scalar temp1 = alpha * actual_x[j]; Scalar temp2 = Scalar(0); - for (int i = std::max(0, j - *k); i < j; ++i) { + for (EIGEN_BLAS_INT i = std::max<EIGEN_BLAS_INT>(0, j - *k); i < j; ++i) { Scalar aij = a[(*k + i - j) + j * *lda]; actual_y[i] += temp1 * aij; temp2 += Eigen::numext::conj(aij) * actual_x[i]; @@ -164,11 +165,11 @@ actual_y[j] += Scalar(Eigen::numext::real(a[*k + j * *lda])) * temp1 + alpha * temp2; } } else { - for (int j = 0; j < *n; ++j) { + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { Scalar temp1 = alpha * actual_x[j]; Scalar temp2 = Scalar(0); actual_y[j] += Scalar(Eigen::numext::real(a[j * *lda])) * temp1; - for (int i = j + 1; i <= std::min(*n - 1, j + *k); ++i) { + for (EIGEN_BLAS_INT i = j + 1; i <= std::min(*n - 1, j + *k); ++i) { Scalar aij = a[(i - j) + j * *lda]; actual_y[i] += temp1 * aij; temp2 += Eigen::numext::conj(aij) * actual_x[i]; @@ -191,15 +192,15 @@ * Diagonal elements are real; off-diagonal contributions use conjugation. */ EIGEN_BLAS_FUNC(hpmv) -(char *uplo, int *n, RealScalar *palpha, RealScalar *pap, RealScalar *px, int *incx, RealScalar *pbeta, RealScalar *py, - int *incy) { +(char *uplo, EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *pap, RealScalar *px, EIGEN_BLAS_INT *incx, + RealScalar *pbeta, RealScalar *py, EIGEN_BLAS_INT *incy) { const Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); const Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); const Scalar *ap = reinterpret_cast<const Scalar *>(pap); const Scalar *x = reinterpret_cast<const Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) @@ -229,10 +230,10 @@ return; } - int kk = 0; + EIGEN_BLAS_INT kk = 0; if (UPLO(*uplo) == UP) { // Upper triangle packed: column j occupies ap[kk..kk+j]. - for (int j = 0; j < *n; ++j) { + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { Scalar temp1 = alpha * actual_x[j]; // Diagonal is real. actual_y[j] += Scalar(Eigen::numext::real(ap[kk + j])) * temp1; @@ -244,8 +245,8 @@ } } else { // Lower triangle packed: column j occupies ap[kk..kk+(n-j-1)]. - for (int j = 0; j < *n; ++j) { - int len = *n - j - 1; + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { + EIGEN_BLAS_INT len = *n - j - 1; Scalar temp1 = alpha * actual_x[j]; // Diagonal is real. actual_y[j] += Scalar(Eigen::numext::real(ap[kk])) * temp1; @@ -268,20 +269,23 @@ * where alpha is a real scalar, x is an n element vector and A is an * n by n hermitian matrix, supplied in packed form. */ -EIGEN_BLAS_FUNC(hpr)(char *uplo, int *n, RealScalar *palpha, RealScalar *px, int *incx, RealScalar *pap) { - typedef void (*functype)(int, Scalar *, const Scalar *, RealScalar); +EIGEN_BLAS_FUNC(hpr) +(char *uplo, EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *pap) { + typedef void (*functype)(EIGEN_BLAS_INT, Scalar *, const Scalar *, RealScalar); static const functype func[2] = { // array index: UP - (Eigen::internal::selfadjoint_packed_rank1_update<Scalar, int, Eigen::ColMajor, Eigen::Upper, false, Conj>::run), + (Eigen::internal::selfadjoint_packed_rank1_update<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, Eigen::Upper, false, + Conj>::run), // array index: LO - (Eigen::internal::selfadjoint_packed_rank1_update<Scalar, int, Eigen::ColMajor, Eigen::Lower, false, Conj>::run), + (Eigen::internal::selfadjoint_packed_rank1_update<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, Eigen::Lower, false, + Conj>::run), }; Scalar *x = reinterpret_cast<Scalar *>(px); Scalar *ap = reinterpret_cast<Scalar *>(pap); RealScalar alpha = *palpha; - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) @@ -310,13 +314,14 @@ * n by n hermitian matrix, supplied in packed form. */ EIGEN_BLAS_FUNC(hpr2) -(char *uplo, int *n, RealScalar *palpha, RealScalar *px, int *incx, RealScalar *py, int *incy, RealScalar *pap) { - typedef void (*functype)(int, Scalar *, const Scalar *, const Scalar *, Scalar); +(char *uplo, EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *py, + EIGEN_BLAS_INT *incy, RealScalar *pap) { + typedef void (*functype)(EIGEN_BLAS_INT, Scalar *, const Scalar *, const Scalar *, Scalar); static const functype func[2] = { // array index: UP - (Eigen::internal::packed_rank2_update_selector<Scalar, int, Eigen::Upper>::run), + (Eigen::internal::packed_rank2_update_selector<Scalar, EIGEN_BLAS_INT, Eigen::Upper>::run), // array index: LO - (Eigen::internal::packed_rank2_update_selector<Scalar, int, Eigen::Lower>::run), + (Eigen::internal::packed_rank2_update_selector<Scalar, EIGEN_BLAS_INT, Eigen::Lower>::run), }; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -324,7 +329,7 @@ Scalar *ap = reinterpret_cast<Scalar *>(pap); Scalar alpha = *reinterpret_cast<Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) @@ -356,27 +361,29 @@ * where alpha is a real scalar, x is an n element vector and A is an * n by n hermitian matrix. */ -EIGEN_BLAS_FUNC(her)(char *uplo, int *n, RealScalar *palpha, RealScalar *px, int *incx, RealScalar *pa, int *lda) { - typedef void (*functype)(int, Scalar *, int, const Scalar *, const Scalar *, const Scalar &); +EIGEN_BLAS_FUNC(her) +(char *uplo, EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *pa, + EIGEN_BLAS_INT *lda) { + typedef void (*functype)(EIGEN_BLAS_INT, Scalar *, EIGEN_BLAS_INT, const Scalar *, const Scalar *, const Scalar &); static const functype func[2] = { // array index: UP - (Eigen::selfadjoint_rank1_update<Scalar, int, Eigen::ColMajor, Eigen::Upper, false, Conj>::run), + (Eigen::selfadjoint_rank1_update<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, Eigen::Upper, false, Conj>::run), // array index: LO - (Eigen::selfadjoint_rank1_update<Scalar, int, Eigen::ColMajor, Eigen::Lower, false, Conj>::run), + (Eigen::selfadjoint_rank1_update<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, Eigen::Lower, false, Conj>::run), }; Scalar *x = reinterpret_cast<Scalar *>(px); Scalar *a = reinterpret_cast<Scalar *>(pa); RealScalar alpha = *reinterpret_cast<RealScalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) info = 2; else if (*incx == 0) info = 5; - else if (*lda < std::max(1, *n)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, *n)) info = 7; if (info) return xerbla_(SCALAR_SUFFIX_UP "HER ", &info, kBlasNameLength); @@ -402,14 +409,14 @@ * by n hermitian matrix. */ EIGEN_BLAS_FUNC(her2) -(char *uplo, int *n, RealScalar *palpha, RealScalar *px, int *incx, RealScalar *py, int *incy, RealScalar *pa, - int *lda) { - typedef void (*functype)(int, Scalar *, int, const Scalar *, const Scalar *, Scalar); +(char *uplo, EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *py, + EIGEN_BLAS_INT *incy, RealScalar *pa, EIGEN_BLAS_INT *lda) { + typedef void (*functype)(EIGEN_BLAS_INT, Scalar *, EIGEN_BLAS_INT, const Scalar *, const Scalar *, Scalar); static const functype func[2] = { // array index: UP - (Eigen::internal::rank2_update_selector<Scalar, int, Eigen::Upper>::run), + (Eigen::internal::rank2_update_selector<Scalar, EIGEN_BLAS_INT, Eigen::Upper>::run), // array index: LO - (Eigen::internal::rank2_update_selector<Scalar, int, Eigen::Lower>::run), + (Eigen::internal::rank2_update_selector<Scalar, EIGEN_BLAS_INT, Eigen::Lower>::run), }; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -417,7 +424,7 @@ Scalar *a = reinterpret_cast<Scalar *>(pa); Scalar alpha = *reinterpret_cast<Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) @@ -426,7 +433,7 @@ info = 5; else if (*incy == 0) info = 7; - else if (*lda < std::max(1, *n)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, *n)) info = 9; if (info) return xerbla_(SCALAR_SUFFIX_UP "HER2 ", &info, kBlasNameLength); @@ -454,13 +461,14 @@ * vector and A is an m by n matrix. */ EIGEN_BLAS_FUNC(geru) -(int *m, int *n, RealScalar *palpha, RealScalar *px, int *incx, RealScalar *py, int *incy, RealScalar *pa, int *lda) { +(EIGEN_BLAS_INT *m, EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *py, + EIGEN_BLAS_INT *incy, RealScalar *pa, EIGEN_BLAS_INT *lda) { Scalar *x = reinterpret_cast<Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); Scalar *a = reinterpret_cast<Scalar *>(pa); Scalar alpha = *reinterpret_cast<Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (*m < 0) info = 1; else if (*n < 0) @@ -469,7 +477,7 @@ info = 5; else if (*incy == 0) info = 7; - else if (*lda < std::max(1, *m)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, *m)) info = 9; if (info) return xerbla_(SCALAR_SUFFIX_UP "GERU ", &info, kBlasNameLength); @@ -478,8 +486,8 @@ Scalar *x_cpy = get_compact_vector(x, *m, *incx); Scalar *y_cpy = get_compact_vector(y, *n, *incy); - Eigen::internal::general_rank1_update<Scalar, int, Eigen::ColMajor, false, false>::run(*m, *n, a, *lda, x_cpy, y_cpy, - alpha); + Eigen::internal::general_rank1_update<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, false, false>::run( + *m, *n, a, *lda, x_cpy, y_cpy, alpha); if (x_cpy != x) delete[] x_cpy; if (y_cpy != y) delete[] y_cpy; @@ -493,13 +501,14 @@ * vector and A is an m by n matrix. */ EIGEN_BLAS_FUNC(gerc) -(int *m, int *n, RealScalar *palpha, RealScalar *px, int *incx, RealScalar *py, int *incy, RealScalar *pa, int *lda) { +(EIGEN_BLAS_INT *m, EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *py, + EIGEN_BLAS_INT *incy, RealScalar *pa, EIGEN_BLAS_INT *lda) { Scalar *x = reinterpret_cast<Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); Scalar *a = reinterpret_cast<Scalar *>(pa); Scalar alpha = *reinterpret_cast<Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (*m < 0) info = 1; else if (*n < 0) @@ -508,7 +517,7 @@ info = 5; else if (*incy == 0) info = 7; - else if (*lda < std::max(1, *m)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, *m)) info = 9; if (info) return xerbla_(SCALAR_SUFFIX_UP "GERC ", &info, kBlasNameLength); @@ -517,8 +526,8 @@ Scalar *x_cpy = get_compact_vector(x, *m, *incx); Scalar *y_cpy = get_compact_vector(y, *n, *incy); - Eigen::internal::general_rank1_update<Scalar, int, Eigen::ColMajor, false, Conj>::run(*m, *n, a, *lda, x_cpy, y_cpy, - alpha); + Eigen::internal::general_rank1_update<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, false, Conj>::run(*m, *n, a, *lda, + x_cpy, y_cpy, alpha); if (x_cpy != x) delete[] x_cpy; if (y_cpy != y) delete[] y_cpy;
diff --git a/blas/level2_impl.h b/blas/level2_impl.h index 3f16f98..9c21ba4 100644 --- a/blas/level2_impl.h +++ b/blas/level2_impl.h
@@ -25,16 +25,18 @@ }; EIGEN_BLAS_FUNC(gemv) -(const char *opa, const int *m, const int *n, const RealScalar *palpha, const RealScalar *pa, const int *lda, - const RealScalar *pb, const int *incb, const RealScalar *pbeta, RealScalar *pc, const int *incc) { - typedef void (*functype)(int, int, const Scalar *, int, const Scalar *, int, Scalar *, int, Scalar); +(const char *opa, const EIGEN_BLAS_INT *m, const EIGEN_BLAS_INT *n, const RealScalar *palpha, const RealScalar *pa, + const EIGEN_BLAS_INT *lda, const RealScalar *pb, const EIGEN_BLAS_INT *incb, const RealScalar *pbeta, RealScalar *pc, + const EIGEN_BLAS_INT *incc) { + typedef void (*functype)(EIGEN_BLAS_INT, EIGEN_BLAS_INT, const Scalar *, EIGEN_BLAS_INT, const Scalar *, + EIGEN_BLAS_INT, Scalar *, EIGEN_BLAS_INT, Scalar); static const functype func[4] = { // array index: NOTR - (general_matrix_vector_product_wrapper<int, Scalar, Eigen::ColMajor, false, false>::run), + (general_matrix_vector_product_wrapper<EIGEN_BLAS_INT, Scalar, Eigen::ColMajor, false, false>::run), // array index: TR - (general_matrix_vector_product_wrapper<int, Scalar, Eigen::RowMajor, false, false>::run), + (general_matrix_vector_product_wrapper<EIGEN_BLAS_INT, Scalar, Eigen::RowMajor, false, false>::run), // array index: ADJ - (general_matrix_vector_product_wrapper<int, Scalar, Eigen::RowMajor, Conj, false>::run), 0}; + (general_matrix_vector_product_wrapper<EIGEN_BLAS_INT, Scalar, Eigen::RowMajor, Conj, false>::run), 0}; const Scalar *a = reinterpret_cast<const Scalar *>(pa); const Scalar *b = reinterpret_cast<const Scalar *>(pb); @@ -43,14 +45,14 @@ Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); // check arguments - int info = 0; + EIGEN_BLAS_INT info = 0; if (OP(*opa) == INVALID) info = 1; else if (*m < 0) info = 2; else if (*n < 0) info = 3; - else if (*lda < std::max(1, *m)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, *m)) info = 6; else if (*incb == 0) info = 8; @@ -60,8 +62,8 @@ if (*m == 0 || *n == 0 || (alpha == Scalar(0) && beta == Scalar(1))) return; - int actual_m = *m; - int actual_n = *n; + EIGEN_BLAS_INT actual_m = *m; + EIGEN_BLAS_INT actual_n = *n; int code = OP(*opa); if (code != NOTR) std::swap(actual_m, actual_n); @@ -84,51 +86,60 @@ } EIGEN_BLAS_FUNC(trsv) -(const char *uplo, const char *opa, const char *diag, const int *n, const RealScalar *pa, const int *lda, - RealScalar *pb, const int *incb) { - typedef void (*functype)(int, const Scalar *, int, Scalar *); +(const char *uplo, const char *opa, const char *diag, const EIGEN_BLAS_INT *n, const RealScalar *pa, + const EIGEN_BLAS_INT *lda, RealScalar *pb, const EIGEN_BLAS_INT *incb) { + typedef void (*functype)(EIGEN_BLAS_INT, const Scalar *, EIGEN_BLAS_INT, Scalar *); using Eigen::ColMajor; using Eigen::Lower; using Eigen::OnTheLeft; using Eigen::RowMajor; using Eigen::UnitDiag; using Eigen::Upper; - static const functype func[16] = { - // array index: NOTR | (UP << 2) | (NUNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | 0, false, ColMajor>::run), - // array index: TR | (UP << 2) | (NUNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | 0, false, RowMajor>::run), - // array index: ADJ | (UP << 2) | (NUNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | 0, Conj, RowMajor>::run), 0, - // array index: NOTR | (LO << 2) | (NUNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | 0, false, ColMajor>::run), - // array index: TR | (LO << 2) | (NUNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | 0, false, RowMajor>::run), - // array index: ADJ | (LO << 2) | (NUNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | 0, Conj, RowMajor>::run), 0, - // array index: NOTR | (UP << 2) | (UNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | UnitDiag, false, - ColMajor>::run), - // array index: TR | (UP << 2) | (UNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | UnitDiag, false, - RowMajor>::run), - // array index: ADJ | (UP << 2) | (UNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | UnitDiag, Conj, RowMajor>::run), - 0, - // array index: NOTR | (LO << 2) | (UNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | UnitDiag, false, - ColMajor>::run), - // array index: TR | (LO << 2) | (UNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | UnitDiag, false, - RowMajor>::run), - // array index: ADJ | (LO << 2) | (UNIT << 3) - (Eigen::internal::triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | UnitDiag, Conj, RowMajor>::run), - 0}; + static const functype func[16] = {// array index: NOTR | (UP << 2) | (NUNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Upper | 0, false, ColMajor>::run), + // array index: TR | (UP << 2) | (NUNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Lower | 0, false, RowMajor>::run), + // array index: ADJ | (UP << 2) | (NUNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Lower | 0, Conj, RowMajor>::run), + 0, + // array index: NOTR | (LO << 2) | (NUNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Lower | 0, false, ColMajor>::run), + // array index: TR | (LO << 2) | (NUNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Upper | 0, false, RowMajor>::run), + // array index: ADJ | (LO << 2) | (NUNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Upper | 0, Conj, RowMajor>::run), + 0, + // array index: NOTR | (UP << 2) | (UNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Upper | UnitDiag, false, ColMajor>::run), + // array index: TR | (UP << 2) | (UNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Lower | UnitDiag, false, RowMajor>::run), + // array index: ADJ | (UP << 2) | (UNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Lower | UnitDiag, Conj, RowMajor>::run), + 0, + // array index: NOTR | (LO << 2) | (UNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Lower | UnitDiag, false, ColMajor>::run), + // array index: TR | (LO << 2) | (UNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Upper | UnitDiag, false, RowMajor>::run), + // array index: ADJ | (LO << 2) | (UNIT << 3) + (Eigen::internal::triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, + Upper | UnitDiag, Conj, RowMajor>::run), + 0}; const Scalar *a = reinterpret_cast<const Scalar *>(pa); Scalar *b = reinterpret_cast<Scalar *>(pb); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (OP(*opa) == INVALID) @@ -137,7 +148,7 @@ info = 3; else if (*n < 0) info = 4; - else if (*lda < std::max(1, *n)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, *n)) info = 6; else if (*incb == 0) info = 8; @@ -152,9 +163,10 @@ } EIGEN_BLAS_FUNC(trmv) -(const char *uplo, const char *opa, const char *diag, const int *n, const RealScalar *pa, const int *lda, - RealScalar *pb, const int *incb) { - typedef void (*functype)(int, int, const Scalar *, int, const Scalar *, int, Scalar *, int, const Scalar &); +(const char *uplo, const char *opa, const char *diag, const EIGEN_BLAS_INT *n, const RealScalar *pa, + const EIGEN_BLAS_INT *lda, RealScalar *pb, const EIGEN_BLAS_INT *incb) { + typedef void (*functype)(EIGEN_BLAS_INT, EIGEN_BLAS_INT, const Scalar *, EIGEN_BLAS_INT, const Scalar *, + EIGEN_BLAS_INT, Scalar *, EIGEN_BLAS_INT, const Scalar &); using Eigen::ColMajor; using Eigen::Lower; using Eigen::OnTheLeft; @@ -163,44 +175,50 @@ using Eigen::Upper; static const functype func[16] = { // array index: NOTR | (UP << 2) | (NUNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Upper | 0, Scalar, false, Scalar, false, ColMajor>::run), + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | 0, Scalar, false, Scalar, false, + ColMajor>::run), // array index: TR | (UP << 2) | (NUNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Lower | 0, Scalar, false, Scalar, false, RowMajor>::run), + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | 0, Scalar, false, Scalar, false, + RowMajor>::run), // array index: ADJ | (UP << 2) | (NUNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Lower | 0, Scalar, Conj, Scalar, false, RowMajor>::run), + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | 0, Scalar, Conj, Scalar, false, + RowMajor>::run), 0, // array index: NOTR | (LO << 2) | (NUNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Lower | 0, Scalar, false, Scalar, false, ColMajor>::run), + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | 0, Scalar, false, Scalar, false, + ColMajor>::run), // array index: TR | (LO << 2) | (NUNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Upper | 0, Scalar, false, Scalar, false, RowMajor>::run), + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | 0, Scalar, false, Scalar, false, + RowMajor>::run), // array index: ADJ | (LO << 2) | (NUNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Upper | 0, Scalar, Conj, Scalar, false, RowMajor>::run), + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | 0, Scalar, Conj, Scalar, false, + RowMajor>::run), 0, // array index: NOTR | (UP << 2) | (UNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Upper | UnitDiag, Scalar, false, Scalar, false, + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | UnitDiag, Scalar, false, Scalar, false, ColMajor>::run), // array index: TR | (UP << 2) | (UNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Lower | UnitDiag, Scalar, false, Scalar, false, + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | UnitDiag, Scalar, false, Scalar, false, RowMajor>::run), // array index: ADJ | (UP << 2) | (UNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Lower | UnitDiag, Scalar, Conj, Scalar, false, + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | UnitDiag, Scalar, Conj, Scalar, false, RowMajor>::run), 0, // array index: NOTR | (LO << 2) | (UNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Lower | UnitDiag, Scalar, false, Scalar, false, + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | UnitDiag, Scalar, false, Scalar, false, ColMajor>::run), // array index: TR | (LO << 2) | (UNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Upper | UnitDiag, Scalar, false, Scalar, false, + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | UnitDiag, Scalar, false, Scalar, false, RowMajor>::run), // array index: ADJ | (LO << 2) | (UNIT << 3) - (Eigen::internal::triangular_matrix_vector_product<int, Upper | UnitDiag, Scalar, Conj, Scalar, false, + (Eigen::internal::triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | UnitDiag, Scalar, Conj, Scalar, false, RowMajor>::run), 0}; const Scalar *a = reinterpret_cast<const Scalar *>(pa); Scalar *b = reinterpret_cast<Scalar *>(pb); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (OP(*opa) == INVALID) @@ -209,7 +227,7 @@ info = 3; else if (*n < 0) info = 4; - else if (*lda < std::max(1, *n)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, *n)) info = 6; else if (*incb == 0) info = 8; @@ -238,16 +256,17 @@ * m by n band matrix, with kl sub-diagonals and ku super-diagonals. */ EIGEN_BLAS_FUNC(gbmv) -(char *trans, int *m, int *n, int *kl, int *ku, RealScalar *palpha, RealScalar *pa, int *lda, RealScalar *px, int *incx, - RealScalar *pbeta, RealScalar *py, int *incy) { +(char *trans, EIGEN_BLAS_INT *m, EIGEN_BLAS_INT *n, EIGEN_BLAS_INT *kl, EIGEN_BLAS_INT *ku, RealScalar *palpha, + RealScalar *pa, EIGEN_BLAS_INT *lda, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *pbeta, RealScalar *py, + EIGEN_BLAS_INT *incy) { const Scalar *a = reinterpret_cast<const Scalar *>(pa); const Scalar *x = reinterpret_cast<const Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); - int coeff_rows = *kl + *ku + 1; + EIGEN_BLAS_INT coeff_rows = *kl + *ku + 1; - int info = 0; + EIGEN_BLAS_INT info = 0; if (OP(*trans) == INVALID) info = 1; else if (*m < 0) @@ -268,8 +287,8 @@ if (*m == 0 || *n == 0 || (alpha == Scalar(0) && beta == Scalar(1))) return; - int actual_m = *m; - int actual_n = *n; + EIGEN_BLAS_INT actual_m = *m; + EIGEN_BLAS_INT actual_n = *n; if (OP(*trans) != NOTR) std::swap(actual_m, actual_n); const Scalar *actual_x = get_compact_vector(x, actual_n, *incx); @@ -284,12 +303,12 @@ ConstMatrixType mat_coeffs(a, coeff_rows, *n, *lda); - int nb = std::min(*n, (*m) + (*ku)); - for (int j = 0; j < nb; ++j) { - int start = std::max(0, j - *ku); - int end = std::min((*m) - 1, j + *kl); - int len = end - start + 1; - int offset = (*ku) - j + start; + EIGEN_BLAS_INT nb = std::min(*n, (*m) + (*ku)); + for (EIGEN_BLAS_INT j = 0; j < nb; ++j) { + EIGEN_BLAS_INT start = std::max<EIGEN_BLAS_INT>(0, j - *ku); + EIGEN_BLAS_INT end = std::min((*m) - 1, j + *kl); + EIGEN_BLAS_INT len = end - start + 1; + EIGEN_BLAS_INT offset = (*ku) - j + start; if (OP(*trans) == NOTR) make_vector(actual_y + start, len) += (alpha * actual_x[j]) * mat_coeffs.col(j).segment(offset, len); else if (OP(*trans) == TR) @@ -315,11 +334,12 @@ * lower triangle stores A[i,j] at a[(i-j) + j*lda]. */ EIGEN_BLAS_FUNC(tbmv) -(char *uplo, char *opa, char *diag, int *n, int *k, RealScalar *pa, int *lda, RealScalar *px, int *incx) { +(char *uplo, char *opa, char *diag, EIGEN_BLAS_INT *n, EIGEN_BLAS_INT *k, RealScalar *pa, EIGEN_BLAS_INT *lda, + RealScalar *px, EIGEN_BLAS_INT *incx) { Scalar *a = reinterpret_cast<Scalar *>(pa); Scalar *x = reinterpret_cast<Scalar *>(px); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (OP(*opa) == INVALID) @@ -349,20 +369,20 @@ ConstMatrixType band(a, *k + 1, *n, *lda); if (op == NOTR) { if (upper) { - for (int j = 0; j < *n; ++j) { + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { if (actual_x[j] != Scalar(0)) { - int start = std::max(0, j - *k); - int len = j - start; - int offset = *k - (j - start); + EIGEN_BLAS_INT start = std::max<EIGEN_BLAS_INT>(0, j - *k); + EIGEN_BLAS_INT len = j - start; + EIGEN_BLAS_INT offset = *k - (j - start); Scalar temp = actual_x[j]; if (len > 0) make_vector(actual_x + start, len) += temp * band.col(j).segment(offset, len); if (!unit) actual_x[j] = temp * band(*k, j); } } } else { - for (int j = *n - 1; j >= 0; --j) { + for (EIGEN_BLAS_INT j = *n - 1; j >= 0; --j) { if (actual_x[j] != Scalar(0)) { - int len = std::min(*n - 1, j + *k) - j; + EIGEN_BLAS_INT len = std::min(*n - 1, j + *k) - j; Scalar temp = actual_x[j]; if (len > 0) make_vector(actual_x + j + 1, len) += temp * band.col(j).segment(1, len); if (!unit) actual_x[j] = temp * band(0, j); @@ -371,10 +391,10 @@ } } else if (op == TR) { if (upper) { - for (int j = *n - 1; j >= 0; --j) { - int start = std::max(0, j - *k); - int len = j - start; - int offset = *k - (j - start); + for (EIGEN_BLAS_INT j = *n - 1; j >= 0; --j) { + EIGEN_BLAS_INT start = std::max<EIGEN_BLAS_INT>(0, j - *k); + EIGEN_BLAS_INT len = j - start; + EIGEN_BLAS_INT offset = *k - (j - start); Scalar temp = actual_x[j]; if (!unit) temp *= band(*k, j); if (len > 0) @@ -382,8 +402,8 @@ actual_x[j] = temp; } } else { - for (int j = 0; j < *n; ++j) { - int len = std::min(*n - 1, j + *k) - j; + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { + EIGEN_BLAS_INT len = std::min(*n - 1, j + *k) - j; Scalar temp = actual_x[j]; if (!unit) temp *= band(0, j); if (len > 0) temp += (band.col(j).segment(1, len).cwiseProduct(make_vector(actual_x + j + 1, len))).sum(); @@ -393,18 +413,18 @@ } else { // Conjugate transpose: .dot() computes conj(lhs) . rhs. if (upper) { - for (int j = *n - 1; j >= 0; --j) { - int start = std::max(0, j - *k); - int len = j - start; - int offset = *k - (j - start); + for (EIGEN_BLAS_INT j = *n - 1; j >= 0; --j) { + EIGEN_BLAS_INT start = std::max<EIGEN_BLAS_INT>(0, j - *k); + EIGEN_BLAS_INT len = j - start; + EIGEN_BLAS_INT offset = *k - (j - start); Scalar temp = actual_x[j]; if (!unit) temp *= Eigen::numext::conj(band(*k, j)); if (len > 0) temp += band.col(j).segment(offset, len).dot(make_vector(actual_x + start, len)); actual_x[j] = temp; } } else { - for (int j = 0; j < *n; ++j) { - int len = std::min(*n - 1, j + *k) - j; + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { + EIGEN_BLAS_INT len = std::min(*n - 1, j + *k) - j; Scalar temp = actual_x[j]; if (!unit) temp *= Eigen::numext::conj(band(0, j)); if (len > 0) temp += band.col(j).segment(1, len).dot(make_vector(actual_x + j + 1, len)); @@ -416,18 +436,20 @@ // Scalar path: for narrow bandwidth, avoid Map overhead. if (op == NOTR) { if (upper) { - for (int j = 0; j < *n; ++j) { + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { if (actual_x[j] != Scalar(0)) { Scalar temp = actual_x[j]; - for (int i = std::max(0, j - *k); i < j; ++i) actual_x[i] += temp * a[(*k + i - j) + j * *lda]; + for (EIGEN_BLAS_INT i = std::max<EIGEN_BLAS_INT>(0, j - *k); i < j; ++i) + actual_x[i] += temp * a[(*k + i - j) + j * *lda]; if (!unit) actual_x[j] = temp * a[*k + j * *lda]; } } } else { - for (int j = *n - 1; j >= 0; --j) { + for (EIGEN_BLAS_INT j = *n - 1; j >= 0; --j) { if (actual_x[j] != Scalar(0)) { Scalar temp = actual_x[j]; - for (int i = j + 1; i <= std::min(*n - 1, j + *k); ++i) actual_x[i] += temp * a[(i - j) + j * *lda]; + for (EIGEN_BLAS_INT i = j + 1; i <= std::min(*n - 1, j + *k); ++i) + actual_x[i] += temp * a[(i - j) + j * *lda]; if (!unit) actual_x[j] = temp * a[j * *lda]; } } @@ -436,17 +458,18 @@ // Transpose or conjugate transpose. auto maybe_conj = [op](Scalar val) -> Scalar { return op == ADJ ? Eigen::numext::conj(val) : val; }; if (upper) { - for (int j = *n - 1; j >= 0; --j) { + for (EIGEN_BLAS_INT j = *n - 1; j >= 0; --j) { Scalar temp = actual_x[j]; if (!unit) temp *= maybe_conj(a[*k + j * *lda]); - for (int i = std::max(0, j - *k); i < j; ++i) temp += maybe_conj(a[(*k + i - j) + j * *lda]) * actual_x[i]; + for (EIGEN_BLAS_INT i = std::max<EIGEN_BLAS_INT>(0, j - *k); i < j; ++i) + temp += maybe_conj(a[(*k + i - j) + j * *lda]) * actual_x[i]; actual_x[j] = temp; } } else { - for (int j = 0; j < *n; ++j) { + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { Scalar temp = actual_x[j]; if (!unit) temp *= maybe_conj(a[j * *lda]); - for (int i = j + 1; i <= std::min(*n - 1, j + *k); ++i) + for (EIGEN_BLAS_INT i = j + 1; i <= std::min(*n - 1, j + *k); ++i) temp += maybe_conj(a[(i - j) + j * *lda]) * actual_x[i]; actual_x[j] = temp; } @@ -469,8 +492,9 @@ * routine. Such tests must be performed before calling this routine. */ EIGEN_BLAS_FUNC(tbsv) -(char *uplo, char *op, char *diag, int *n, int *k, RealScalar *pa, int *lda, RealScalar *px, int *incx) { - typedef void (*functype)(int, int, const Scalar *, int, Scalar *); +(char *uplo, char *op, char *diag, EIGEN_BLAS_INT *n, EIGEN_BLAS_INT *k, RealScalar *pa, EIGEN_BLAS_INT *lda, + RealScalar *px, EIGEN_BLAS_INT *incx) { + typedef void (*functype)(EIGEN_BLAS_INT, EIGEN_BLAS_INT, const Scalar *, EIGEN_BLAS_INT, Scalar *); using Eigen::ColMajor; using Eigen::Lower; using Eigen::OnTheLeft; @@ -479,40 +503,50 @@ using Eigen::Upper; static const functype func[16] = { // array index: NOTR | (UP << 2) | (NUNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Upper | 0, Scalar, false, Scalar, ColMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Upper | 0, Scalar, false, Scalar, + ColMajor>::run), // array index: TR | (UP << 2) | (NUNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Lower | 0, Scalar, false, Scalar, RowMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Lower | 0, Scalar, false, Scalar, + RowMajor>::run), // array index: ADJ | (UP << 2) | (NUNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Lower | 0, Scalar, Conj, Scalar, RowMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Lower | 0, Scalar, Conj, Scalar, RowMajor>::run), 0, // array index: NOTR | (LO << 2) | (NUNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Lower | 0, Scalar, false, Scalar, ColMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Lower | 0, Scalar, false, Scalar, + ColMajor>::run), // array index: TR | (LO << 2) | (NUNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Upper | 0, Scalar, false, Scalar, RowMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Upper | 0, Scalar, false, Scalar, + RowMajor>::run), // array index: ADJ | (LO << 2) | (NUNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Upper | 0, Scalar, Conj, Scalar, RowMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Upper | 0, Scalar, Conj, Scalar, RowMajor>::run), 0, // array index: NOTR | (UP << 2) | (UNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Upper | UnitDiag, Scalar, false, Scalar, ColMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Upper | UnitDiag, Scalar, false, Scalar, + ColMajor>::run), // array index: TR | (UP << 2) | (UNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Lower | UnitDiag, Scalar, false, Scalar, RowMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Lower | UnitDiag, Scalar, false, Scalar, + RowMajor>::run), // array index: ADJ | (UP << 2) | (UNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Lower | UnitDiag, Scalar, Conj, Scalar, RowMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Lower | UnitDiag, Scalar, Conj, Scalar, + RowMajor>::run), 0, // array index: NOTR | (LO << 2) | (UNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Lower | UnitDiag, Scalar, false, Scalar, ColMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Lower | UnitDiag, Scalar, false, Scalar, + ColMajor>::run), // array index: TR | (LO << 2) | (UNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Upper | UnitDiag, Scalar, false, Scalar, RowMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Upper | UnitDiag, Scalar, false, Scalar, + RowMajor>::run), // array index: ADJ | (LO << 2) | (UNIT << 3) - (Eigen::internal::band_solve_triangular_selector<int, Upper | UnitDiag, Scalar, Conj, Scalar, RowMajor>::run), + (Eigen::internal::band_solve_triangular_selector<EIGEN_BLAS_INT, Upper | UnitDiag, Scalar, Conj, Scalar, + RowMajor>::run), 0, }; Scalar *a = reinterpret_cast<Scalar *>(pa); Scalar *x = reinterpret_cast<Scalar *>(px); - int coeff_rows = *k + 1; + EIGEN_BLAS_INT coeff_rows = *k + 1; - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (OP(*op) == INVALID) @@ -531,7 +565,7 @@ if (*n == 0 || (*k == 0 && DIAG(*diag) == UNIT)) return; - int actual_n = *n; + EIGEN_BLAS_INT actual_n = *n; Scalar *actual_x = get_compact_vector(x, actual_n, *incx); @@ -550,8 +584,9 @@ * where x is an n element vector and A is an n by n unit, or non-unit, * upper or lower triangular matrix, supplied in packed form. */ -EIGEN_BLAS_FUNC(tpmv)(char *uplo, char *opa, char *diag, int *n, RealScalar *pap, RealScalar *px, int *incx) { - typedef void (*functype)(int, const Scalar *, const Scalar *, Scalar *, Scalar); +EIGEN_BLAS_FUNC(tpmv) +(char *uplo, char *opa, char *diag, EIGEN_BLAS_INT *n, RealScalar *pap, RealScalar *px, EIGEN_BLAS_INT *incx) { + typedef void (*functype)(EIGEN_BLAS_INT, const Scalar *, const Scalar *, Scalar *, Scalar); using Eigen::ColMajor; using Eigen::Lower; using Eigen::OnTheLeft; @@ -560,50 +595,50 @@ using Eigen::Upper; static const functype func[16] = { // array index: NOTR | (UP << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Upper | 0, Scalar, false, Scalar, false, + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | 0, Scalar, false, Scalar, false, ColMajor>::run), // array index: TR | (UP << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Lower | 0, Scalar, false, Scalar, false, + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | 0, Scalar, false, Scalar, false, RowMajor>::run), // array index: ADJ | (UP << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Lower | 0, Scalar, Conj, Scalar, false, + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | 0, Scalar, Conj, Scalar, false, RowMajor>::run), 0, // array index: NOTR | (LO << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Lower | 0, Scalar, false, Scalar, false, + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | 0, Scalar, false, Scalar, false, ColMajor>::run), // array index: TR | (LO << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Upper | 0, Scalar, false, Scalar, false, + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | 0, Scalar, false, Scalar, false, RowMajor>::run), // array index: ADJ | (LO << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Upper | 0, Scalar, Conj, Scalar, false, + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | 0, Scalar, Conj, Scalar, false, RowMajor>::run), 0, // array index: NOTR | (UP << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Upper | UnitDiag, Scalar, false, Scalar, false, - ColMajor>::run), + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | UnitDiag, Scalar, false, Scalar, + false, ColMajor>::run), // array index: TR | (UP << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Lower | UnitDiag, Scalar, false, Scalar, false, - RowMajor>::run), + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | UnitDiag, Scalar, false, Scalar, + false, RowMajor>::run), // array index: ADJ | (UP << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Lower | UnitDiag, Scalar, Conj, Scalar, false, - RowMajor>::run), + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | UnitDiag, Scalar, Conj, Scalar, + false, RowMajor>::run), 0, // array index: NOTR | (LO << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Lower | UnitDiag, Scalar, false, Scalar, false, - ColMajor>::run), + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Lower | UnitDiag, Scalar, false, Scalar, + false, ColMajor>::run), // array index: TR | (LO << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Upper | UnitDiag, Scalar, false, Scalar, false, - RowMajor>::run), + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | UnitDiag, Scalar, false, Scalar, + false, RowMajor>::run), // array index: ADJ | (LO << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_matrix_vector_product<int, Upper | UnitDiag, Scalar, Conj, Scalar, false, - RowMajor>::run), + (Eigen::internal::packed_triangular_matrix_vector_product<EIGEN_BLAS_INT, Upper | UnitDiag, Scalar, Conj, Scalar, + false, RowMajor>::run), 0}; Scalar *ap = reinterpret_cast<Scalar *>(pap); Scalar *x = reinterpret_cast<Scalar *>(px); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (OP(*opa) == INVALID) @@ -641,8 +676,9 @@ * No test for singularity or near-singularity is included in this * routine. Such tests must be performed before calling this routine. */ -EIGEN_BLAS_FUNC(tpsv)(char *uplo, char *opa, char *diag, int *n, RealScalar *pap, RealScalar *px, int *incx) { - typedef void (*functype)(int, const Scalar *, Scalar *); +EIGEN_BLAS_FUNC(tpsv) +(char *uplo, char *opa, char *diag, EIGEN_BLAS_INT *n, RealScalar *pap, RealScalar *px, EIGEN_BLAS_INT *incx) { + typedef void (*functype)(EIGEN_BLAS_INT, const Scalar *, Scalar *); using Eigen::ColMajor; using Eigen::Lower; using Eigen::OnTheLeft; @@ -651,48 +687,50 @@ using Eigen::Upper; static const functype func[16] = { // array index: NOTR | (UP << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | 0, false, + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Upper | 0, false, ColMajor>::run), // array index: TR | (UP << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | 0, false, + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Lower | 0, false, RowMajor>::run), // array index: ADJ | (UP << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | 0, Conj, RowMajor>::run), + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Lower | 0, Conj, + RowMajor>::run), 0, // array index: NOTR | (LO << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | 0, false, + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Lower | 0, false, ColMajor>::run), // array index: TR | (LO << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | 0, false, + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Upper | 0, false, RowMajor>::run), // array index: ADJ | (LO << 2) | (NUNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | 0, Conj, RowMajor>::run), + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Upper | 0, Conj, + RowMajor>::run), 0, // array index: NOTR | (UP << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | UnitDiag, false, - ColMajor>::run), + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Upper | UnitDiag, + false, ColMajor>::run), // array index: TR | (UP << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | UnitDiag, false, - RowMajor>::run), + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Lower | UnitDiag, + false, RowMajor>::run), // array index: ADJ | (UP << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | UnitDiag, Conj, - RowMajor>::run), + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Lower | UnitDiag, + Conj, RowMajor>::run), 0, // array index: NOTR | (LO << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Lower | UnitDiag, false, - ColMajor>::run), + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Lower | UnitDiag, + false, ColMajor>::run), // array index: TR | (LO << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | UnitDiag, false, - RowMajor>::run), + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Upper | UnitDiag, + false, RowMajor>::run), // array index: ADJ | (LO << 2) | (UNIT << 3) - (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, int, OnTheLeft, Upper | UnitDiag, Conj, - RowMajor>::run), + (Eigen::internal::packed_triangular_solve_vector<Scalar, Scalar, EIGEN_BLAS_INT, OnTheLeft, Upper | UnitDiag, + Conj, RowMajor>::run), 0}; Scalar *ap = reinterpret_cast<Scalar *>(pap); Scalar *x = reinterpret_cast<Scalar *>(px); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (OP(*opa) == INVALID)
diff --git a/blas/level2_real_impl.h b/blas/level2_real_impl.h index 64265aa..b9f03cc 100644 --- a/blas/level2_real_impl.h +++ b/blas/level2_real_impl.h
@@ -12,17 +12,18 @@ // y = alpha*A*x + beta*y EIGEN_BLAS_FUNC(symv) -(const char *uplo, const int *n, const RealScalar *palpha, const RealScalar *pa, const int *lda, const RealScalar *px, - const int *incx, const RealScalar *pbeta, RealScalar *py, const int *incy) { - typedef void (*functype)(int, const Scalar *, int, const Scalar *, Scalar *, Scalar); +(const char *uplo, const EIGEN_BLAS_INT *n, const RealScalar *palpha, const RealScalar *pa, const EIGEN_BLAS_INT *lda, + const RealScalar *px, const EIGEN_BLAS_INT *incx, const RealScalar *pbeta, RealScalar *py, + const EIGEN_BLAS_INT *incy) { + typedef void (*functype)(EIGEN_BLAS_INT, const Scalar *, EIGEN_BLAS_INT, const Scalar *, Scalar *, Scalar); using Eigen::ColMajor; using Eigen::Lower; using Eigen::Upper; static const functype func[2] = { // array index: UP - (Eigen::internal::selfadjoint_matrix_vector_product<Scalar, int, ColMajor, Upper, false, false>::run), + (Eigen::internal::selfadjoint_matrix_vector_product<Scalar, EIGEN_BLAS_INT, ColMajor, Upper, false, false>::run), // array index: LO - (Eigen::internal::selfadjoint_matrix_vector_product<Scalar, int, ColMajor, Lower, false, false>::run), + (Eigen::internal::selfadjoint_matrix_vector_product<Scalar, EIGEN_BLAS_INT, ColMajor, Lower, false, false>::run), }; const Scalar *a = reinterpret_cast<const Scalar *>(pa); @@ -32,12 +33,12 @@ Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); // check arguments - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) info = 2; - else if (*lda < std::max(1, *n)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, *n)) info = 5; else if (*incx == 0) info = 7; @@ -68,31 +69,31 @@ // C := alpha*x*x' + C EIGEN_BLAS_FUNC(syr) -(const char *uplo, const int *n, const RealScalar *palpha, const RealScalar *px, const int *incx, RealScalar *pc, - const int *ldc) { - typedef void (*functype)(int, Scalar *, int, const Scalar *, const Scalar *, const Scalar &); +(const char *uplo, const EIGEN_BLAS_INT *n, const RealScalar *palpha, const RealScalar *px, const EIGEN_BLAS_INT *incx, + RealScalar *pc, const EIGEN_BLAS_INT *ldc) { + typedef void (*functype)(EIGEN_BLAS_INT, Scalar *, EIGEN_BLAS_INT, const Scalar *, const Scalar *, const Scalar &); using Eigen::ColMajor; using Eigen::Lower; using Eigen::Upper; static const functype func[2] = { // array index: UP - (Eigen::selfadjoint_rank1_update<Scalar, int, ColMajor, Upper, false, Conj>::run), + (Eigen::selfadjoint_rank1_update<Scalar, EIGEN_BLAS_INT, ColMajor, Upper, false, Conj>::run), // array index: LO - (Eigen::selfadjoint_rank1_update<Scalar, int, ColMajor, Lower, false, Conj>::run), + (Eigen::selfadjoint_rank1_update<Scalar, EIGEN_BLAS_INT, ColMajor, Lower, false, Conj>::run), }; const Scalar *x = reinterpret_cast<const Scalar *>(px); Scalar *c = reinterpret_cast<Scalar *>(pc); Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) info = 2; else if (*incx == 0) info = 5; - else if (*ldc < std::max(1, *n)) + else if (*ldc < std::max<EIGEN_BLAS_INT>(1, *n)) info = 7; if (info) return xerbla_(SCALAR_SUFFIX_UP "SYR ", &info, kBlasNameLength); @@ -111,14 +112,14 @@ // C := alpha*x*y' + alpha*y*x' + C EIGEN_BLAS_FUNC(syr2) -(const char *uplo, const int *n, const RealScalar *palpha, const RealScalar *px, const int *incx, const RealScalar *py, - const int *incy, RealScalar *pc, const int *ldc) { - typedef void (*functype)(int, Scalar *, int, const Scalar *, const Scalar *, Scalar); +(const char *uplo, const EIGEN_BLAS_INT *n, const RealScalar *palpha, const RealScalar *px, const EIGEN_BLAS_INT *incx, + const RealScalar *py, const EIGEN_BLAS_INT *incy, RealScalar *pc, const EIGEN_BLAS_INT *ldc) { + typedef void (*functype)(EIGEN_BLAS_INT, Scalar *, EIGEN_BLAS_INT, const Scalar *, const Scalar *, Scalar); static const functype func[2] = { // array index: UP - (Eigen::internal::rank2_update_selector<Scalar, int, Eigen::Upper>::run), + (Eigen::internal::rank2_update_selector<Scalar, EIGEN_BLAS_INT, Eigen::Upper>::run), // array index: LO - (Eigen::internal::rank2_update_selector<Scalar, int, Eigen::Lower>::run), + (Eigen::internal::rank2_update_selector<Scalar, EIGEN_BLAS_INT, Eigen::Lower>::run), }; const Scalar *x = reinterpret_cast<const Scalar *>(px); @@ -126,7 +127,7 @@ Scalar *c = reinterpret_cast<Scalar *>(pc); Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) @@ -135,7 +136,7 @@ info = 5; else if (*incy == 0) info = 7; - else if (*ldc < std::max(1, *n)) + else if (*ldc < std::max<EIGEN_BLAS_INT>(1, *n)) info = 9; if (info) return xerbla_(SCALAR_SUFFIX_UP "SYR2 ", &info, kBlasNameLength); @@ -170,15 +171,15 @@ * lower triangle stores A[i,j] at a[(i-j) + j*lda]. */ EIGEN_BLAS_FUNC(sbmv) -(char *uplo, int *n, int *k, RealScalar *palpha, RealScalar *pa, int *lda, RealScalar *px, int *incx, RealScalar *pbeta, - RealScalar *py, int *incy) { +(char *uplo, EIGEN_BLAS_INT *n, EIGEN_BLAS_INT *k, RealScalar *palpha, RealScalar *pa, EIGEN_BLAS_INT *lda, + RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *pbeta, RealScalar *py, EIGEN_BLAS_INT *incy) { const Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); const Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); const Scalar *a = reinterpret_cast<const Scalar *>(pa); const Scalar *x = reinterpret_cast<const Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) @@ -216,10 +217,10 @@ // Vectorized path: use Eigen Map segments for the inner band operations. ConstMatrixType band(a, *k + 1, *n, *lda); if (UPLO(*uplo) == UP) { - for (int j = 0; j < *n; ++j) { - int start = std::max(0, j - *k); - int len = j - start; - int offset = *k - (j - start); + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { + EIGEN_BLAS_INT start = std::max<EIGEN_BLAS_INT>(0, j - *k); + EIGEN_BLAS_INT len = j - start; + EIGEN_BLAS_INT offset = *k - (j - start); Scalar temp1 = alpha * actual_x[j]; actual_y[j] += temp1 * band(*k, j); if (len > 0) { @@ -228,8 +229,8 @@ } } } else { - for (int j = 0; j < *n; ++j) { - int len = std::min(*n - 1, j + *k) - j; + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { + EIGEN_BLAS_INT len = std::min(*n - 1, j + *k) - j; Scalar temp1 = alpha * actual_x[j]; actual_y[j] += temp1 * band(0, j); if (len > 0) { @@ -241,10 +242,10 @@ } else { // Scalar path: for narrow bandwidth, avoid Map overhead. if (UPLO(*uplo) == UP) { - for (int j = 0; j < *n; ++j) { + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { Scalar temp1 = alpha * actual_x[j]; Scalar temp2 = Scalar(0); - for (int i = std::max(0, j - *k); i < j; ++i) { + for (EIGEN_BLAS_INT i = std::max<EIGEN_BLAS_INT>(0, j - *k); i < j; ++i) { Scalar aij = a[(*k + i - j) + j * *lda]; actual_y[i] += temp1 * aij; temp2 += aij * actual_x[i]; @@ -252,11 +253,11 @@ actual_y[j] += temp1 * a[*k + j * *lda] + alpha * temp2; } } else { - for (int j = 0; j < *n; ++j) { + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { Scalar temp1 = alpha * actual_x[j]; Scalar temp2 = Scalar(0); actual_y[j] += temp1 * a[j * *lda]; - for (int i = j + 1; i <= std::min(*n - 1, j + *k); ++i) { + for (EIGEN_BLAS_INT i = j + 1; i <= std::min(*n - 1, j + *k); ++i) { Scalar aij = a[(i - j) + j * *lda]; actual_y[i] += temp1 * aij; temp2 += aij * actual_x[i]; @@ -282,15 +283,15 @@ * lower triangle stores column j at positions kk..kk+(n-j-1). */ EIGEN_BLAS_FUNC(spmv) -(char *uplo, int *n, RealScalar *palpha, RealScalar *pap, RealScalar *px, int *incx, RealScalar *pbeta, RealScalar *py, - int *incy) { +(char *uplo, EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *pap, RealScalar *px, EIGEN_BLAS_INT *incx, + RealScalar *pbeta, RealScalar *py, EIGEN_BLAS_INT *incy) { const Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); const Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); const Scalar *ap = reinterpret_cast<const Scalar *>(pap); const Scalar *x = reinterpret_cast<const Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) @@ -320,10 +321,10 @@ return; } - int kk = 0; + EIGEN_BLAS_INT kk = 0; if (UPLO(*uplo) == UP) { // Upper triangle packed: column j occupies ap[kk..kk+j]. - for (int j = 0; j < *n; ++j) { + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { Scalar temp1 = alpha * actual_x[j]; actual_y[j] += temp1 * ap[kk + j]; if (j > 0) { @@ -334,8 +335,8 @@ } } else { // Lower triangle packed: column j occupies ap[kk..kk+(n-j-1)]. - for (int j = 0; j < *n; ++j) { - int len = *n - j - 1; + for (EIGEN_BLAS_INT j = 0; j < *n; ++j) { + EIGEN_BLAS_INT len = *n - j - 1; Scalar temp1 = alpha * actual_x[j]; actual_y[j] += temp1 * ap[kk]; if (len > 0) { @@ -357,20 +358,22 @@ * where alpha is a real scalar, x is an n element vector and A is an * n by n symmetric matrix, supplied in packed form. */ -EIGEN_BLAS_FUNC(spr)(char *uplo, int *n, Scalar *palpha, Scalar *px, int *incx, Scalar *pap) { - typedef void (*functype)(int, Scalar *, const Scalar *, Scalar); +EIGEN_BLAS_FUNC(spr)(char *uplo, EIGEN_BLAS_INT *n, Scalar *palpha, Scalar *px, EIGEN_BLAS_INT *incx, Scalar *pap) { + typedef void (*functype)(EIGEN_BLAS_INT, Scalar *, const Scalar *, Scalar); static const functype func[2] = { // array index: UP - (Eigen::internal::selfadjoint_packed_rank1_update<Scalar, int, Eigen::ColMajor, Eigen::Upper, false, false>::run), + (Eigen::internal::selfadjoint_packed_rank1_update<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, Eigen::Upper, false, + false>::run), // array index: LO - (Eigen::internal::selfadjoint_packed_rank1_update<Scalar, int, Eigen::ColMajor, Eigen::Lower, false, false>::run), + (Eigen::internal::selfadjoint_packed_rank1_update<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, Eigen::Lower, false, + false>::run), }; Scalar *x = reinterpret_cast<Scalar *>(px); Scalar *ap = reinterpret_cast<Scalar *>(pap); Scalar alpha = *reinterpret_cast<Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) @@ -399,13 +402,14 @@ * n by n symmetric matrix, supplied in packed form. */ EIGEN_BLAS_FUNC(spr2) -(char *uplo, int *n, RealScalar *palpha, RealScalar *px, int *incx, RealScalar *py, int *incy, RealScalar *pap) { - typedef void (*functype)(int, Scalar *, const Scalar *, const Scalar *, Scalar); +(char *uplo, EIGEN_BLAS_INT *n, RealScalar *palpha, RealScalar *px, EIGEN_BLAS_INT *incx, RealScalar *py, + EIGEN_BLAS_INT *incy, RealScalar *pap) { + typedef void (*functype)(EIGEN_BLAS_INT, Scalar *, const Scalar *, const Scalar *, Scalar); static const functype func[2] = { // array index: UP - (Eigen::internal::packed_rank2_update_selector<Scalar, int, Eigen::Upper>::run), + (Eigen::internal::packed_rank2_update_selector<Scalar, EIGEN_BLAS_INT, Eigen::Upper>::run), // array index: LO - (Eigen::internal::packed_rank2_update_selector<Scalar, int, Eigen::Lower>::run), + (Eigen::internal::packed_rank2_update_selector<Scalar, EIGEN_BLAS_INT, Eigen::Lower>::run), }; Scalar *x = reinterpret_cast<Scalar *>(px); @@ -413,7 +417,7 @@ Scalar *ap = reinterpret_cast<Scalar *>(pap); Scalar alpha = *reinterpret_cast<Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (*n < 0) @@ -446,13 +450,14 @@ * vector and A is an m by n matrix. */ EIGEN_BLAS_FUNC(ger) -(int *m, int *n, Scalar *palpha, Scalar *px, int *incx, Scalar *py, int *incy, Scalar *pa, int *lda) { +(EIGEN_BLAS_INT *m, EIGEN_BLAS_INT *n, Scalar *palpha, Scalar *px, EIGEN_BLAS_INT *incx, Scalar *py, + EIGEN_BLAS_INT *incy, Scalar *pa, EIGEN_BLAS_INT *lda) { Scalar *x = reinterpret_cast<Scalar *>(px); Scalar *y = reinterpret_cast<Scalar *>(py); Scalar *a = reinterpret_cast<Scalar *>(pa); Scalar alpha = *reinterpret_cast<Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (*m < 0) info = 1; else if (*n < 0) @@ -461,7 +466,7 @@ info = 5; else if (*incy == 0) info = 7; - else if (*lda < std::max(1, *m)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, *m)) info = 9; if (info) return xerbla_(SCALAR_SUFFIX_UP "GER ", &info, kBlasNameLength); @@ -470,8 +475,8 @@ Scalar *x_cpy = get_compact_vector(x, *m, *incx); Scalar *y_cpy = get_compact_vector(y, *n, *incy); - Eigen::internal::general_rank1_update<Scalar, int, Eigen::ColMajor, false, false>::run(*m, *n, a, *lda, x_cpy, y_cpy, - alpha); + Eigen::internal::general_rank1_update<Scalar, EIGEN_BLAS_INT, Eigen::ColMajor, false, false>::run( + *m, *n, a, *lda, x_cpy, y_cpy, alpha); if (x_cpy != x) delete[] x_cpy; if (y_cpy != y) delete[] y_cpy;
diff --git a/blas/level3_impl.h b/blas/level3_impl.h index 94c8a08..dedc27c 100644 --- a/blas/level3_impl.h +++ b/blas/level3_impl.h
@@ -10,9 +10,9 @@ #include "common.h" EIGEN_BLAS_FUNC(gemm) -(const char *opa, const char *opb, const int *m, const int *n, const int *k, const RealScalar *palpha, - const RealScalar *pa, const int *lda, const RealScalar *pb, const int *ldb, const RealScalar *pbeta, RealScalar *pc, - const int *ldc) { +(const char *opa, const char *opb, const EIGEN_BLAS_INT *m, const EIGEN_BLAS_INT *n, const EIGEN_BLAS_INT *k, + const RealScalar *palpha, const RealScalar *pa, const EIGEN_BLAS_INT *lda, const RealScalar *pb, + const EIGEN_BLAS_INT *ldb, const RealScalar *pbeta, RealScalar *pc, const EIGEN_BLAS_INT *ldc) { using Eigen::ColMajor; using Eigen::DenseIndex; using Eigen::Dynamic; @@ -58,7 +58,7 @@ Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); - int info = 0; + EIGEN_BLAS_INT info = 0; if (OP(*opa) == INVALID) info = 1; else if (OP(*opb) == INVALID) @@ -69,11 +69,11 @@ info = 4; else if (*k < 0) info = 5; - else if (*lda < std::max(1, (OP(*opa) == NOTR) ? *m : *k)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, (OP(*opa) == NOTR) ? *m : *k)) info = 8; - else if (*ldb < std::max(1, (OP(*opb) == NOTR) ? *k : *n)) + else if (*ldb < std::max<EIGEN_BLAS_INT>(1, (OP(*opb) == NOTR) ? *k : *n)) info = 10; - else if (*ldc < std::max(1, *m)) + else if (*ldc < std::max<EIGEN_BLAS_INT>(1, *m)) info = 13; if (info) return xerbla_(SCALAR_SUFFIX_UP "GEMM ", &info, kBlasNameLength); @@ -96,9 +96,9 @@ } EIGEN_BLAS_FUNC(gemmtr) -(const char *uplo, const char *opa, const char *opb, const int *n, const int *k, const RealScalar *palpha, - const RealScalar *pa, const int *lda, const RealScalar *pb, const int *ldb, const RealScalar *pbeta, RealScalar *pc, - const int *ldc) { +(const char *uplo, const char *opa, const char *opb, const EIGEN_BLAS_INT *n, const EIGEN_BLAS_INT *k, + const RealScalar *palpha, const RealScalar *pa, const EIGEN_BLAS_INT *lda, const RealScalar *pb, + const EIGEN_BLAS_INT *ldb, const RealScalar *pbeta, RealScalar *pc, const EIGEN_BLAS_INT *ldc) { using Eigen::ColMajor; using Eigen::DenseIndex; using Eigen::Dynamic; @@ -179,7 +179,7 @@ Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (OP(*opa) == INVALID) @@ -190,11 +190,11 @@ info = 4; else if (*k < 0) info = 5; - else if (*lda < std::max(1, (OP(*opa) == NOTR) ? *n : *k)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, (OP(*opa) == NOTR) ? *n : *k)) info = 8; - else if (*ldb < std::max(1, (OP(*opb) == NOTR) ? *k : *n)) + else if (*ldb < std::max<EIGEN_BLAS_INT>(1, (OP(*opb) == NOTR) ? *k : *n)) info = 10; - else if (*ldc < std::max(1, *n)) + else if (*ldc < std::max<EIGEN_BLAS_INT>(1, *n)) info = 13; if (info) return xerbla_(SCALAR_SUFFIX_UP "GEMMTR", &info, kBlasNameLength + 1); @@ -228,8 +228,9 @@ } EIGEN_BLAS_FUNC(trsm) -(const char *side, const char *uplo, const char *opa, const char *diag, const int *m, const int *n, - const RealScalar *palpha, const RealScalar *pa, const int *lda, RealScalar *pb, const int *ldb) { +(const char *side, const char *uplo, const char *opa, const char *diag, const EIGEN_BLAS_INT *m, + const EIGEN_BLAS_INT *n, const RealScalar *palpha, const RealScalar *pa, const EIGEN_BLAS_INT *lda, RealScalar *pb, + const EIGEN_BLAS_INT *ldb) { using Eigen::ColMajor; using Eigen::DenseIndex; using Eigen::Dynamic; @@ -327,7 +328,7 @@ Scalar *b = reinterpret_cast<Scalar *>(pb); Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (SIDE(*side) == INVALID) info = 1; else if (UPLO(*uplo) == INVALID) @@ -340,9 +341,9 @@ info = 5; else if (*n < 0) info = 6; - else if (*lda < std::max(1, (SIDE(*side) == LEFT) ? *m : *n)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, (SIDE(*side) == LEFT) ? *m : *n)) info = 9; - else if (*ldb < std::max(1, *m)) + else if (*ldb < std::max<EIGEN_BLAS_INT>(1, *m)) info = 11; if (info) return xerbla_(SCALAR_SUFFIX_UP "TRSM ", &info, kBlasNameLength); @@ -366,8 +367,9 @@ // b = alpha*op(a)*b for side = 'L'or'l' // b = alpha*b*op(a) for side = 'R'or'r' EIGEN_BLAS_FUNC(trmm) -(const char *side, const char *uplo, const char *opa, const char *diag, const int *m, const int *n, - const RealScalar *palpha, const RealScalar *pa, const int *lda, RealScalar *pb, const int *ldb) { +(const char *side, const char *uplo, const char *opa, const char *diag, const EIGEN_BLAS_INT *m, + const EIGEN_BLAS_INT *n, const RealScalar *palpha, const RealScalar *pa, const EIGEN_BLAS_INT *lda, RealScalar *pb, + const EIGEN_BLAS_INT *ldb) { using Eigen::ColMajor; using Eigen::DenseIndex; using Eigen::Dynamic; @@ -464,7 +466,7 @@ Scalar *b = reinterpret_cast<Scalar *>(pb); Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); - int info = 0; + EIGEN_BLAS_INT info = 0; if (SIDE(*side) == INVALID) info = 1; else if (UPLO(*uplo) == INVALID) @@ -477,9 +479,9 @@ info = 5; else if (*n < 0) info = 6; - else if (*lda < std::max(1, (SIDE(*side) == LEFT) ? *m : *n)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, (SIDE(*side) == LEFT) ? *m : *n)) info = 9; - else if (*ldb < std::max(1, *m)) + else if (*ldb < std::max<EIGEN_BLAS_INT>(1, *m)) info = 11; if (info) return xerbla_(SCALAR_SUFFIX_UP "TRMM ", &info, kBlasNameLength); @@ -505,15 +507,16 @@ // c = alpha*a*b + beta*c for side = 'L'or'l' // c = alpha*b*a + beta*c for side = 'R'or'r EIGEN_BLAS_FUNC(symm) -(const char *side, const char *uplo, const int *m, const int *n, const RealScalar *palpha, const RealScalar *pa, - const int *lda, const RealScalar *pb, const int *ldb, const RealScalar *pbeta, RealScalar *pc, const int *ldc) { +(const char *side, const char *uplo, const EIGEN_BLAS_INT *m, const EIGEN_BLAS_INT *n, const RealScalar *palpha, + const RealScalar *pa, const EIGEN_BLAS_INT *lda, const RealScalar *pb, const EIGEN_BLAS_INT *ldb, + const RealScalar *pbeta, RealScalar *pc, const EIGEN_BLAS_INT *ldc) { const Scalar *a = reinterpret_cast<const Scalar *>(pa); const Scalar *b = reinterpret_cast<const Scalar *>(pb); Scalar *c = reinterpret_cast<Scalar *>(pc); Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); - int info = 0; + EIGEN_BLAS_INT info = 0; if (SIDE(*side) == INVALID) info = 1; else if (UPLO(*uplo) == INVALID) @@ -522,11 +525,11 @@ info = 3; else if (*n < 0) info = 4; - else if (*lda < std::max(1, (SIDE(*side) == LEFT) ? *m : *n)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, (SIDE(*side) == LEFT) ? *m : *n)) info = 7; - else if (*ldb < std::max(1, *m)) + else if (*ldb < std::max<EIGEN_BLAS_INT>(1, *m)) info = 9; - else if (*ldc < std::max(1, *m)) + else if (*ldc < std::max<EIGEN_BLAS_INT>(1, *m)) info = 12; if (info) return xerbla_(SCALAR_SUFFIX_UP "SYMM ", &info, kBlasNameLength); @@ -539,7 +542,7 @@ if (*m == 0 || *n == 0) return; - int size = (SIDE(*side) == LEFT) ? (*m) : (*n); + EIGEN_BLAS_INT size = (SIDE(*side) == LEFT) ? (*m) : (*n); using Eigen::ColMajor; using Eigen::DenseIndex; using Eigen::Dynamic; @@ -594,8 +597,8 @@ // c = alpha*a*a' + beta*c for op = 'N'or'n' // c = alpha*a'*a + beta*c for op = 'T'or't','C'or'c' EIGEN_BLAS_FUNC(syrk) -(const char *uplo, const char *op, const int *n, const int *k, const RealScalar *palpha, const RealScalar *pa, - const int *lda, const RealScalar *pbeta, RealScalar *pc, const int *ldc) { +(const char *uplo, const char *op, const EIGEN_BLAS_INT *n, const EIGEN_BLAS_INT *k, const RealScalar *palpha, + const RealScalar *pa, const EIGEN_BLAS_INT *lda, const RealScalar *pbeta, RealScalar *pc, const EIGEN_BLAS_INT *ldc) { using Eigen::ColMajor; using Eigen::DenseIndex; using Eigen::Dynamic; @@ -633,7 +636,7 @@ Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (OP(*op) == INVALID || (ISCOMPLEX && OP(*op) == ADJ)) @@ -642,9 +645,9 @@ info = 3; else if (*k < 0) info = 4; - else if (*lda < std::max(1, (OP(*op) == NOTR) ? *n : *k)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, (OP(*op) == NOTR) ? *n : *k)) info = 7; - else if (*ldc < std::max(1, *n)) + else if (*ldc < std::max<EIGEN_BLAS_INT>(1, *n)) info = 10; if (info) return xerbla_(SCALAR_SUFFIX_UP "SYRK ", &info, kBlasNameLength); @@ -691,15 +694,16 @@ // c = alpha*a*b' + alpha*b*a' + beta*c for op = 'N'or'n' // c = alpha*a'*b + alpha*b'*a + beta*c for op = 'T'or't' EIGEN_BLAS_FUNC(syr2k) -(const char *uplo, const char *op, const int *n, const int *k, const RealScalar *palpha, const RealScalar *pa, - const int *lda, const RealScalar *pb, const int *ldb, const RealScalar *pbeta, RealScalar *pc, const int *ldc) { +(const char *uplo, const char *op, const EIGEN_BLAS_INT *n, const EIGEN_BLAS_INT *k, const RealScalar *palpha, + const RealScalar *pa, const EIGEN_BLAS_INT *lda, const RealScalar *pb, const EIGEN_BLAS_INT *ldb, + const RealScalar *pbeta, RealScalar *pc, const EIGEN_BLAS_INT *ldc) { const Scalar *a = reinterpret_cast<const Scalar *>(pa); const Scalar *b = reinterpret_cast<const Scalar *>(pb); Scalar *c = reinterpret_cast<Scalar *>(pc); Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if (OP(*op) == INVALID || (ISCOMPLEX && OP(*op) == ADJ)) @@ -708,11 +712,11 @@ info = 3; else if (*k < 0) info = 4; - else if (*lda < std::max(1, (OP(*op) == NOTR) ? *n : *k)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, (OP(*op) == NOTR) ? *n : *k)) info = 7; - else if (*ldb < std::max(1, (OP(*op) == NOTR) ? *n : *k)) + else if (*ldb < std::max<EIGEN_BLAS_INT>(1, (OP(*op) == NOTR) ? *n : *k)) info = 9; - else if (*ldc < std::max(1, *n)) + else if (*ldc < std::max<EIGEN_BLAS_INT>(1, *n)) info = 12; if (info) return xerbla_(SCALAR_SUFFIX_UP "SYR2K", &info, kBlasNameLength); @@ -758,15 +762,16 @@ // c = alpha*a*b + beta*c for side = 'L'or'l' // c = alpha*b*a + beta*c for side = 'R'or'r EIGEN_BLAS_FUNC(hemm) -(const char *side, const char *uplo, const int *m, const int *n, const RealScalar *palpha, const RealScalar *pa, - const int *lda, const RealScalar *pb, const int *ldb, const RealScalar *pbeta, RealScalar *pc, const int *ldc) { +(const char *side, const char *uplo, const EIGEN_BLAS_INT *m, const EIGEN_BLAS_INT *n, const RealScalar *palpha, + const RealScalar *pa, const EIGEN_BLAS_INT *lda, const RealScalar *pb, const EIGEN_BLAS_INT *ldb, + const RealScalar *pbeta, RealScalar *pc, const EIGEN_BLAS_INT *ldc) { const Scalar *a = reinterpret_cast<const Scalar *>(pa); const Scalar *b = reinterpret_cast<const Scalar *>(pb); Scalar *c = reinterpret_cast<Scalar *>(pc); Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); Scalar beta = *reinterpret_cast<const Scalar *>(pbeta); - int info = 0; + EIGEN_BLAS_INT info = 0; if (SIDE(*side) == INVALID) info = 1; else if (UPLO(*uplo) == INVALID) @@ -775,11 +780,11 @@ info = 3; else if (*n < 0) info = 4; - else if (*lda < std::max(1, (SIDE(*side) == LEFT) ? *m : *n)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, (SIDE(*side) == LEFT) ? *m : *n)) info = 7; - else if (*ldb < std::max(1, *m)) + else if (*ldb < std::max<EIGEN_BLAS_INT>(1, *m)) info = 9; - else if (*ldc < std::max(1, *m)) + else if (*ldc < std::max<EIGEN_BLAS_INT>(1, *m)) info = 12; if (info) return xerbla_(SCALAR_SUFFIX_UP "HEMM ", &info, kBlasNameLength); @@ -796,7 +801,7 @@ using Eigen::RowMajor; using Eigen::Upper; - int size = (SIDE(*side) == LEFT) ? (*m) : (*n); + EIGEN_BLAS_INT size = (SIDE(*side) == LEFT) ? (*m) : (*n); Eigen::internal::gemm_blocking_space<ColMajor, Scalar, Scalar, Dynamic, Dynamic, Dynamic> blocking(*m, *n, size, 1, false); @@ -833,8 +838,8 @@ // c = alpha*a*conj(a') + beta*c for op = 'N'or'n' // c = alpha*conj(a')*a + beta*c for op = 'C'or'c' EIGEN_BLAS_FUNC(herk) -(const char *uplo, const char *op, const int *n, const int *k, const RealScalar *palpha, const RealScalar *pa, - const int *lda, const RealScalar *pbeta, RealScalar *pc, const int *ldc) { +(const char *uplo, const char *op, const EIGEN_BLAS_INT *n, const EIGEN_BLAS_INT *k, const RealScalar *palpha, + const RealScalar *pa, const EIGEN_BLAS_INT *lda, const RealScalar *pbeta, RealScalar *pc, const EIGEN_BLAS_INT *ldc) { using Eigen::ColMajor; using Eigen::DenseIndex; using Eigen::Dynamic; @@ -868,7 +873,7 @@ RealScalar alpha = *palpha; RealScalar beta = *pbeta; - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if ((OP(*op) == INVALID) || (OP(*op) == TR)) @@ -877,9 +882,9 @@ info = 3; else if (*k < 0) info = 4; - else if (*lda < std::max(1, (OP(*op) == NOTR) ? *n : *k)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, (OP(*op) == NOTR) ? *n : *k)) info = 7; - else if (*ldc < std::max(1, *n)) + else if (*ldc < std::max<EIGEN_BLAS_INT>(1, *n)) info = 10; if (info) return xerbla_(SCALAR_SUFFIX_UP "HERK ", &info, kBlasNameLength); @@ -913,15 +918,16 @@ // c = alpha*a*conj(b') + conj(alpha)*b*conj(a') + beta*c, for op = 'N'or'n' // c = alpha*conj(a')*b + conj(alpha)*conj(b')*a + beta*c, for op = 'C'or'c' EIGEN_BLAS_FUNC(her2k) -(const char *uplo, const char *op, const int *n, const int *k, const RealScalar *palpha, const RealScalar *pa, - const int *lda, const RealScalar *pb, const int *ldb, const RealScalar *pbeta, RealScalar *pc, const int *ldc) { +(const char *uplo, const char *op, const EIGEN_BLAS_INT *n, const EIGEN_BLAS_INT *k, const RealScalar *palpha, + const RealScalar *pa, const EIGEN_BLAS_INT *lda, const RealScalar *pb, const EIGEN_BLAS_INT *ldb, + const RealScalar *pbeta, RealScalar *pc, const EIGEN_BLAS_INT *ldc) { const Scalar *a = reinterpret_cast<const Scalar *>(pa); const Scalar *b = reinterpret_cast<const Scalar *>(pb); Scalar *c = reinterpret_cast<Scalar *>(pc); Scalar alpha = *reinterpret_cast<const Scalar *>(palpha); RealScalar beta = *pbeta; - int info = 0; + EIGEN_BLAS_INT info = 0; if (UPLO(*uplo) == INVALID) info = 1; else if ((OP(*op) == INVALID) || (OP(*op) == TR)) @@ -930,11 +936,11 @@ info = 3; else if (*k < 0) info = 4; - else if (*lda < std::max(1, (OP(*op) == NOTR) ? *n : *k)) + else if (*lda < std::max<EIGEN_BLAS_INT>(1, (OP(*op) == NOTR) ? *n : *k)) info = 7; - else if (*ldb < std::max(1, (OP(*op) == NOTR) ? *n : *k)) + else if (*ldb < std::max<EIGEN_BLAS_INT>(1, (OP(*op) == NOTR) ? *n : *k)) info = 9; - else if (*ldc < std::max(1, *n)) + else if (*ldc < std::max<EIGEN_BLAS_INT>(1, *n)) info = 12; if (info) return xerbla_(SCALAR_SUFFIX_UP "HER2K", &info, kBlasNameLength);
diff --git a/blas/single.cpp b/blas/single.cpp index 6928c05..3341bf1 100644 --- a/blas/single.cpp +++ b/blas/single.cpp
@@ -19,6 +19,7 @@ #include "level2_real_impl.h" #include "level3_impl.h" -float EIGEN_BLAS_FUNC_NAME(dsdot)(int* n, float* alpha, float* x, int* incx, float* y, int* incy) { +float EIGEN_BLAS_FUNC_NAME(dsdot)(EIGEN_BLAS_INT* n, float* alpha, float* x, EIGEN_BLAS_INT* incx, float* y, + EIGEN_BLAS_INT* incy) { return double(*alpha) + BLASFUNC(dsdot)(n, x, incx, y, incy); }
diff --git a/blas/testing/CMakeLists.txt b/blas/testing/CMakeLists.txt index 6f72e07..7f9a78e 100644 --- a/blas/testing/CMakeLists.txt +++ b/blas/testing/CMakeLists.txt
@@ -10,6 +10,14 @@ set(EIGEN_Fortran_COMPILER_WORKS OFF) endif() +if(EIGEN_BUILD_BLAS_ILP64) + if(TARGET eigen_blas_ilp64) + set(eigen_blas_ilp64_lib eigen_blas_ilp64) + else() + set(eigen_blas_ilp64_lib eigen_blas_ilp64_static) + endif() +endif() + if (EIGEN_Fortran_COMPILER_WORKS) macro(ei_add_blas_test testname) @@ -59,5 +67,77 @@ target_compile_options(zblat3 PRIVATE "-O0") endif() +if(EIGEN_BUILD_BLAS_ILP64) + include(CheckFortranCompilerFlag) + set(EIGEN_FORTRAN_INT64_FLAG "") + foreach(flag "-fdefault-integer-8" "-i8") + string(REGEX REPLACE "[^A-Za-z0-9]" "_" flag_var "${flag}") + check_fortran_compiler_flag("${flag}" EIGEN_FORTRAN_HAS_${flag_var}) + if(EIGEN_FORTRAN_HAS_${flag_var}) + set(EIGEN_FORTRAN_INT64_FLAG "${flag}") + break() + endif() + endforeach() + + if(EIGEN_FORTRAN_INT64_FLAG) + set(ilp64_test_dir "${CMAKE_CURRENT_BINARY_DIR}/ilp64") + file(MAKE_DIRECTORY "${ilp64_test_dir}") + + macro(ei_add_blas_ilp64_test testname) + set(targetname "${testname}_64") + set(filename "${testname}.f") + add_executable(${targetname} ${filename}) + target_compile_options(${targetname} PRIVATE "${EIGEN_FORTRAN_INT64_FLAG}") + set_target_properties(${targetname} PROPERTIES + OUTPUT_NAME "${testname}" + RUNTIME_OUTPUT_DIRECTORY "${ilp64_test_dir}") + target_link_libraries(${targetname} ${eigen_blas_ilp64_lib}) + if(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO) + target_link_libraries(${targetname} ${EIGEN_STANDARD_LIBRARIES_TO_LINK_TO}) + endif() + target_link_libraries(${targetname} ${EXTERNAL_LIBS}) + add_test(NAME ${targetname} + COMMAND "${Eigen_SOURCE_DIR}/blas/testing/runblastest.sh" "${testname}" "${Eigen_SOURCE_DIR}/blas/testing/${testname}.dat" + WORKING_DIRECTORY "${ilp64_test_dir}") + add_dependencies(buildtests ${targetname}) + endmacro() + + ei_add_blas_ilp64_test(sblat1) + ei_add_blas_ilp64_test(sblat2) + ei_add_blas_ilp64_test(sblat3) + + ei_add_blas_ilp64_test(dblat1) + ei_add_blas_ilp64_test(dblat2) + ei_add_blas_ilp64_test(dblat3) + + ei_add_blas_ilp64_test(cblat1) + ei_add_blas_ilp64_test(cblat2) + ei_add_blas_ilp64_test(cblat3) + + ei_add_blas_ilp64_test(zblat1) + ei_add_blas_ilp64_test(zblat2) + ei_add_blas_ilp64_test(zblat3) + + if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU") + target_compile_options(cblat3_64 PRIVATE "-O0") + target_compile_options(zblat3_64 PRIVATE "-O0") + endif() + endif() +endif() + +endif() + +if(EIGEN_BUILD_BLAS_ILP64) + add_executable(blas_ilp64 blas_ilp64.cpp) + target_include_directories(blas_ilp64 PRIVATE "${Eigen_SOURCE_DIR}" "${Eigen_SOURCE_DIR}/test") + target_compile_definitions(blas_ilp64 PRIVATE "EIGEN_USE_BLAS" "EIGEN_64BIT_BLAS") + target_link_libraries(blas_ilp64 ${eigen_blas_ilp64_lib}) + if(EIGEN_STANDARD_LIBRARIES_TO_LINK_TO) + target_link_libraries(blas_ilp64 ${EIGEN_STANDARD_LIBRARIES_TO_LINK_TO}) + endif() + # Windows has no rpath: the loader finds the DLL only beside the executable or on PATH. + set_target_properties(blas_ilp64 PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$<TARGET_FILE_DIR:${eigen_blas_ilp64_lib}>") + add_test(NAME blas_ilp64 COMMAND blas_ilp64) + add_dependencies(buildtests blas_ilp64) endif()
diff --git a/blas/testing/blas_ilp64.cpp b/blas/testing/blas_ilp64.cpp new file mode 100644 index 0000000..d720370 --- /dev/null +++ b/blas/testing/blas_ilp64.cpp
@@ -0,0 +1,142 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include "main.h" +#include "../blas.h" + +static_assert(sizeof(EIGEN_BLAS_INT) == 8, "EIGEN_BLAS_INT must be 64-bit in blas_ilp64"); +static_assert(sizeof(Eigen::BlasIndex) == 8, "Eigen::BlasIndex must be 64-bit in blas_ilp64"); + +namespace { + +EIGEN_BLAS_INT g_last_xerbla_info = 0; + +} // namespace + +extern "C" void BLASFUNC(xerbla)(const char* /*msg*/, EIGEN_BLAS_INT* info, size_t /*len*/) { + g_last_xerbla_info = *info; +} + +namespace { + +// Low 32 bits equal 1, full 64-bit value is negative (-4294967295). +// A routine reading 32-bit int* sees 1 and executes; a 64-bit routine rejects it via xerbla_. +constexpr EIGEN_BLAS_INT kNegativeWithPositiveLowWord = + static_cast<EIGEN_BLAS_INT>(std::uint64_t(1) | (std::uint64_t(0xFFFFFFFF) << 32)); + +// A Windows DLL binds its own xerbla_ at link time, so the definition above never sees the +// library's calls there; c == -999 still shows that the argument was rejected. +#if defined(_WIN32) || defined(__CYGWIN__) +constexpr bool kXerblaOverridable = false; +#else +constexpr bool kXerblaOverridable = true; +#endif + +void test_ilp64_upper_word_sensitivity() { + EIGEN_BLAS_INT one = 1; + EIGEN_BLAS_INT bad = kNegativeWithPositiveLowWord; + double alpha = 1.0; + double beta = 0.0; + double a = 3.0; + double b = 5.0; + double c = -999.0; + + g_last_xerbla_info = 0; + BLASFUNC(dgemm)("N", "N", &bad, &one, &one, &alpha, &a, &one, &b, &one, &beta, &c, &one); + if (kXerblaOverridable) VERIFY_IS_EQUAL(g_last_xerbla_info, EIGEN_BLAS_INT(3)); + VERIFY_IS_EQUAL(c, -999.0); + + g_last_xerbla_info = 0; + BLASFUNC(dgemv)("N", &bad, &one, &alpha, &a, &one, &b, &one, &beta, &c, &one); + if (kXerblaOverridable) VERIFY_IS_EQUAL(g_last_xerbla_info, EIGEN_BLAS_INT(2)); + VERIFY_IS_EQUAL(c, -999.0); + + g_last_xerbla_info = 0; + char uplo = 'U'; + BLASFUNC(dsymv)(&uplo, &bad, &alpha, &a, &one, &b, &one, &beta, &c, &one); + if (kXerblaOverridable) VERIFY_IS_EQUAL(g_last_xerbla_info, EIGEN_BLAS_INT(2)); + VERIFY_IS_EQUAL(c, -999.0); + + g_last_xerbla_info = 0; + BLASFUNC(dsyrk)(&uplo, "N", &bad, &one, &alpha, &a, &one, &beta, &c, &one); + if (kXerblaOverridable) VERIFY_IS_EQUAL(g_last_xerbla_info, EIGEN_BLAS_INT(3)); + VERIFY_IS_EQUAL(c, -999.0); +} + +void test_ilp64_level1() { + EIGEN_BLAS_INT n = 5; + EIGEN_BLAS_INT inc_pos = 1; + EIGEN_BLAS_INT inc_neg = -1; + EIGEN_BLAS_INT inc_zero = 0; + + double x[5] = {1.0, -4.0, 2.0, 9.0, -3.0}; + double y[5] = {10.0, 20.0, 30.0, 40.0, 50.0}; + + VERIFY_IS_EQUAL(BLASFUNC(idamax)(&n, x, &inc_pos), EIGEN_BLAS_INT(4)); + VERIFY_IS_EQUAL(BLASFUNC(idamin)(&n, x, &inc_pos), EIGEN_BLAS_INT(1)); + + // Reverse stride (-1 has 0xFFFFFFFF in the upper 32 bits). + double d = BLASFUNC(ddot)(&n, x, &inc_neg, y, &inc_pos); + double expected_d = x[4] * y[0] + x[3] * y[1] + x[2] * y[2] + x[1] * y[3] + x[0] * y[4]; + VERIFY_IS_APPROX(d, expected_d); + + // Broadcast copy (incx == 0). + double src = 7.5; + double dst[5] = {0.0, 0.0, 0.0, 0.0, 0.0}; + BLASFUNC(dcopy)(&n, &src, &inc_zero, dst, &inc_pos); + for (EIGEN_BLAS_INT i = 0; i < n; ++i) { + VERIFY_IS_EQUAL(dst[i], src); + } +} + +template <typename Scalar> +void test_ilp64_eigen_blas_dispatch() { + using Mat = Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic>; + using Vec = Eigen::Matrix<Scalar, Eigen::Dynamic, 1>; + + const Eigen::Index m = 19; + const Eigen::Index n = 23; + const Eigen::Index k = 17; + + Mat A = Mat::Random(m, k); + Mat B = Mat::Random(k, n); + Mat C = Mat::Zero(m, n); + Mat C_ref = A.lazyProduct(B); + C.noalias() = A * B; + VERIFY_IS_APPROX(C, C_ref); + + Vec v = Vec::Random(k); + Vec y = A * v; + Vec y_ref = A.lazyProduct(v); + VERIFY_IS_APPROX(y, y_ref); + + Mat S = Mat::Random(m, m); + S = S + S.adjoint().eval(); + Mat B_sym = Mat::Random(m, n); + Mat C_sym = S.template selfadjointView<Eigen::Upper>() * B_sym; + Mat S_full = S.template selfadjointView<Eigen::Upper>(); + VERIFY_IS_APPROX(C_sym, S_full.lazyProduct(B_sym)); + + Mat T = Mat::Random(m, m); + T.diagonal().array() += Scalar(m); + Mat X = T.template triangularView<Eigen::Upper>().solve(B_sym); + Mat T_upper = T.template triangularView<Eigen::Upper>(); + VERIFY_IS_APPROX(T_upper * X, B_sym); +} + +} // namespace + +EIGEN_DECLARE_TEST(blas_ilp64) { + CALL_SUBTEST(test_ilp64_upper_word_sensitivity()); + CALL_SUBTEST(test_ilp64_level1()); + CALL_SUBTEST(test_ilp64_eigen_blas_dispatch<float>()); + CALL_SUBTEST(test_ilp64_eigen_blas_dispatch<double>()); + CALL_SUBTEST(test_ilp64_eigen_blas_dispatch<std::complex<float>>()); + CALL_SUBTEST(test_ilp64_eigen_blas_dispatch<std::complex<double>>()); +}
diff --git a/blas/xerbla.cpp b/blas/xerbla.cpp index 5a735cc..5853a05 100644 --- a/blas/xerbla.cpp +++ b/blas/xerbla.cpp
@@ -15,8 +15,8 @@ extern "C" { #endif -EIGEN_WEAK_LINKING void xerbla_(const char* msg, int* info, size_t len) { - printf("Eigen BLAS ERROR #%i: %.*s\n", *info, static_cast<int>(len), msg); +EIGEN_WEAK_LINKING void BLASFUNC(xerbla)(const char* msg, EIGEN_BLAS_INT* info, size_t len) { + printf("Eigen BLAS ERROR #%i: %.*s\n", static_cast<int>(*info), static_cast<int>(len), msg); } #ifdef __cplusplus
diff --git a/cmake/EigenConfigureTesting.cmake b/cmake/EigenConfigureTesting.cmake index 4638c26..2d31bca 100644 --- a/cmake/EigenConfigureTesting.cmake +++ b/cmake/EigenConfigureTesting.cmake
@@ -25,6 +25,8 @@ # test/ and contrib/test/ as a parent-scope variable, and ei_add_test_internal # checks it on every test it registers. include(EigenSmokeTestList) +# Split-test parts that ei_add_test compiles together; see the file for the limits. +include(EigenTestPartGroups) if (NOT EIGEN_CTEST_ARGS) # By default, run tests in parallel on all available cores. @@ -82,5 +84,5 @@ endif() elseif(MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D_CRT_SECURE_NO_WARNINGS /D_SCL_SECURE_NO_WARNINGS") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS") endif()
diff --git a/cmake/EigenTestPartGroups.cmake b/cmake/EigenTestPartGroups.cmake new file mode 100644 index 0000000..2e30999 --- /dev/null +++ b/cmake/EigenTestPartGroups.cmake
@@ -0,0 +1,162 @@ +# SPDX-FileCopyrightText: The Eigen Authors +# SPDX-License-Identifier: MPL-2.0 + +# Split-test parts that ei_add_test compiles as one executable per range +# "<first>-<last>" (see cmake/EigenTesting.cmake). Each part repeats the +# frontend work of parsing and instantiating Eigen, so cheap parts compile +# faster together. +# +# Limits every range was checked against: +# - peak RSS of its compile about 4 GiB at most with g++-10, clang++-14 and +# aarch64-linux-gnu-g++-10 at the CI's Release flags (aarch64 g++-10 is the +# hungriest: five ranges reach 4.06-4.29 GiB there); +# - summed runtime of its parts <= 300 s on every configuration of the weekly +# pipeline, half the CI's 600 s retry timeout; +# - no part named in EigenSmokeTestList.cmake, and no test with +# EIGEN_TEST_PART_<N> markers (ei_add_test rejects both). +# Adding or renumbering subtests inside a range grows its compile: re-measure +# the range, or split it, before relying on it. + +set(ei_test_part_groups_adjoint "2-19" "20-24") +set(ei_test_part_groups_array_cwise "1-4" "5-6" "10-12" "13-16" "17-26" "27-30" "31-34" "35-38" "40-41") +set(ei_test_part_groups_array_for_matrix "2-9") +set(ei_test_part_groups_array_replicate "2-8") +set(ei_test_part_groups_array_reverse "2-11") +set(ei_test_part_groups_autodiff "2-6") +set(ei_test_part_groups_autodiff_scalar "2-7") +set(ei_test_part_groups_basicstuff "1-4" "5-8") +set(ei_test_part_groups_bicgstab "1-2" "3-6") +set(ei_test_part_groups_blasutil "2-3" "4-6" "7-8") +set(ei_test_part_groups_block "1-4" "6-10") +set(ei_test_part_groups_block_sparse_matrix "3-8" "10-17" "18-19") +set(ei_test_part_groups_bunchkaufman "1-4" "6-7") +set(ei_test_part_groups_cholesky "7-9") +set(ei_test_part_groups_clz "1-4") +set(ei_test_part_groups_commainitializer "1-2") +set(ei_test_part_groups_complex_components "1-3") +set(ei_test_part_groups_complex_qz "1-3" "4-6" "7-8" "9-10") +set(ei_test_part_groups_concat "1-15") +set(ei_test_part_groups_condition_estimator "1-4" "5-6") +set(ei_test_part_groups_conjugate_gradient "1-2" "3-6") +set(ei_test_part_groups_conservative_resize "2-6") +set(ei_test_part_groups_corners "2-5") +set(ei_test_part_groups_determinant "2-10") +set(ei_test_part_groups_dgmres "2-4" "5-6") +set(ei_test_part_groups_diagonal "3-5") +set(ei_test_part_groups_diagonalmatrices "2-5" "6-9" "10-11") +set(ei_test_part_groups_diagonalview "1-2") +set(ei_test_part_groups_eigensolver_complex "2-5") +set(ei_test_part_groups_eigensolver_generalized_real "2-3" "4-5") +set(ei_test_part_groups_eigensolver_generic "3-5") +set(ei_test_part_groups_EulerAngles "2-5") +set(ei_test_part_groups_fastmath "1-4") +set(ei_test_part_groups_geo_alignedbox "3-14") +set(ei_test_part_groups_geo_homogeneous "2-3") +set(ei_test_part_groups_geo_hyperplane "2-5") +set(ei_test_part_groups_geo_misc "1-10") +set(ei_test_part_groups_geo_orthomethods "2-6") +set(ei_test_part_groups_geo_parametrizedline "2-4") +set(ei_test_part_groups_geo_quaternion "1-3") +set(ei_test_part_groups_geo_transformations "1-2" "3-4" "5-6" "8-9") +set(ei_test_part_groups_gmres "1-3") +set(ei_test_part_groups_hessenberg "2-5") +set(ei_test_part_groups_householder_fastmath "1-2") +set(ei_test_part_groups_idrs "1-2") +set(ei_test_part_groups_incomplete_cholesky "1-2" "3-4") +set(ei_test_part_groups_incomplete_LUT "1-5") +set(ei_test_part_groups_initializer_list_construction "1-6") +set(ei_test_part_groups_inplace_decomposition "2-3" "4-5" "6-7") +set(ei_test_part_groups_integer_types "2-9") +set(ei_test_part_groups_inverse "2-5" "6-8") +set(ei_test_part_groups_jacobi "2-8") +set(ei_test_part_groups_jacobisvd "53-54") +set(ei_test_part_groups_jacobisvd_blocked "1-4" "5-6") +set(ei_test_part_groups_linearstructure "2-12") +set(ei_test_part_groups_lscg "1-5") +set(ei_test_part_groups_lsmr "2-6" "7-11") +set(ei_test_part_groups_lu "1-2" "3-4" "6-8" "10-16") +set(ei_test_part_groups_mapped_matrix "2-13") +set(ei_test_part_groups_mapstaticmethods "2-8") +set(ei_test_part_groups_mapstride "2-6") +set(ei_test_part_groups_matrix_cwise "1-2" "4-5") +set(ei_test_part_groups_matrix_exponential "1-2" "3-4" "5-6" "7-8") +set(ei_test_part_groups_matrix_power "2-3") +set(ei_test_part_groups_matrix_square_root "2-3") +set(ei_test_part_groups_miscmatrices "2-3") +set(ei_test_part_groups_nesting_ops "2-4") +set(ei_test_part_groups_no_automatic_resizing "1-3") +set(ei_test_part_groups_nomalloc "2-3" "4-8" "9-10") +set(ei_test_part_groups_nullary "2-12") +set(ei_test_part_groups_packetmath "1-15") +set(ei_test_part_groups_packetmath_gather "1-2") +set(ei_test_part_groups_polynomialutils "2-9") +set(ei_test_part_groups_prec_inverse_4x4 "2-3") +set(ei_test_part_groups_product_evaluators "1-4") +set(ei_test_part_groups_product_large "1-2" "3-4" "5-6" "7-8" "10-11") +set(ei_test_part_groups_product_mmtr "1-6") +set(ei_test_part_groups_product_notemporary "1-2" "3-5") +set(ei_test_part_groups_product_selfadjoint "2-9") +set(ei_test_part_groups_product_small "1-2" "3-6" "8-10" "11-13" "21-23" "31-32" "33-41" "42-51" "52-55") +set(ei_test_part_groups_product_symm "2-7" "8-10") +set(ei_test_part_groups_product_syrk "2-7") +set(ei_test_part_groups_product_trmv "2-6") +set(ei_test_part_groups_product_trsolve "1-4" "6-12" "13-17") +set(ei_test_part_groups_qr "2-4" "5-9" "11-12") +set(ei_test_part_groups_qr_colpivoting "1-2" "8-11") +set(ei_test_part_groups_qr_fullpivoting "1-3" "5-8") +set(ei_test_part_groups_qr_rand_colpivoting "6-9") +set(ei_test_part_groups_rand "2-15") +set(ei_test_part_groups_random_matrix "1-4" "5-7" "8-9" "10-11" "12-14" "15-16") +set(ei_test_part_groups_real_qz "2-5") +set(ei_test_part_groups_realview "7-8" "9-12") +set(ei_test_part_groups_redux "2-13") +set(ei_test_part_groups_ref "2-10") +set(ei_test_part_groups_reshape "1-7") +set(ei_test_part_groups_rvalue_types "2-5") +set(ei_test_part_groups_scaled_permutation "1-5") +set(ei_test_part_groups_schur_complex "2-5" "8-9" "10-11") +set(ei_test_part_groups_selfadjoint "2-7") +set(ei_test_part_groups_simplicial_cholesky "23-32") +set(ei_test_part_groups_skew_symmetric_matrix3 "1-3") +set(ei_test_part_groups_sparse_basic "4-5") +set(ei_test_part_groups_sparse_block "2-6") +set(ei_test_part_groups_sparse_extra "1-3" "5-6") +set(ei_test_part_groups_sparse_ordering "1-4") +set(ei_test_part_groups_sparse_product "1-2" "5-6") +set(ei_test_part_groups_sparse_ref "2-4") +set(ei_test_part_groups_sparse_scatter "1-3") +set(ei_test_part_groups_sparse_threaded_product "3-4") +set(ei_test_part_groups_sparselu "4-6" "7-10") +set(ei_test_part_groups_sparseqr "1-7") +set(ei_test_part_groups_special_functions "2-5") +set(ei_test_part_groups_special_members "1-2") +set(ei_test_part_groups_special_packetmath "2-4") +set(ei_test_part_groups_stable_norm "2-4" "5-7" "8-11" "12-13") +set(ei_test_part_groups_stable_norm_fastmath "1-2") +set(ei_test_part_groups_stddeque "2-5") +set(ei_test_part_groups_stddeque_overload "2-5") +set(ei_test_part_groups_stdlist "2-5") +set(ei_test_part_groups_stdlist_overload "2-5") +set(ei_test_part_groups_stdvector "2-5") +set(ei_test_part_groups_stdvector_overload "2-5") +set(ei_test_part_groups_structured_bccb "1-2" "5-7") +set(ei_test_part_groups_structured_bindings "1-5") +set(ei_test_part_groups_structured_cauchy "1-2" "3-4") +set(ei_test_part_groups_structured_dplr "3-4") +set(ei_test_part_groups_structured_dpr1 "4-5") +set(ei_test_part_groups_structured_norm "1-5") +set(ei_test_part_groups_structured_vandermonde "1-3") +set(ei_test_part_groups_structured_vandermonde_int_index "1-2") +set(ei_test_part_groups_swap "2-4") +set(ei_test_part_groups_tensor_image_patch "1-12") +set(ei_test_part_groups_tensor_intdiv "1-7") +set(ei_test_part_groups_tensor_uint128 "1-7") +set(ei_test_part_groups_threads_runqueue "1-3") +set(ei_test_part_groups_triangular_in_place "1-6" "7-10") +set(ei_test_part_groups_tridiagonal_eigensolver "4-5") +set(ei_test_part_groups_trsm_packet_cancellation "1-2") +set(ei_test_part_groups_umeyama "1-2" "4-7") +set(ei_test_part_groups_upperbidiagonalization "1-3" "4-5" "6-7" "8-10" "11-13") +set(ei_test_part_groups_vectorwise_norm "1-9" "10-13") +set(ei_test_part_groups_vectorwiseop "2-4" "5-8" "9-12") +set(ei_test_part_groups_visitor "2-14")
diff --git a/cmake/EigenTesting.cmake b/cmake/EigenTesting.cmake index 93333a0..43d5fdf 100644 --- a/cmake/EigenTesting.cmake +++ b/cmake/EigenTesting.cmake
@@ -51,7 +51,7 @@ set(test_compile_options "") if(MSVC) - list(APPEND test_compile_options "/bigobj") + list(APPEND test_compile_options "-bigobj") endif() # let the user pass flags. @@ -63,11 +63,15 @@ if(is_gpu_test AND MSVC AND EIGEN_GPU_TEST_MODE STREQUAL "cuda-language" AND CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA") - # nvcc treats bare /options as input files. Definitions must reach both compilation passes. + # These options are host (MSVC) flags in either spelling: nvcc reads a bare + # /option as an input file and rejects MSVC's -options, so both go through + # -Xcompiler. Definitions must reach both compilation passes. nvcc's own + # options belong in CMAKE_CUDA_FLAGS. foreach(option IN LISTS test_compile_options) - if(option MATCHES "^/D(.+)") + if(option MATCHES "^[/-]D(.+)") target_compile_definitions(${targetname} PRIVATE "${CMAKE_MATCH_1}") - elseif(option MATCHES "^/") + elseif(option MATCHES "^[/-]") + string(REGEX REPLACE "^/" "-" option "${option}") target_compile_options(${targetname} PRIVATE "-Xcompiler=${option}") else() target_compile_options(${targetname} PRIVATE "${option}") @@ -175,6 +179,13 @@ # Moreover, targets <testname> are still generated, they # have the effect of building all the parts of the test. # +# A test listed in EigenTestPartGroups.cmake compiles each listed range of +# parts "<first>-<last>" as one executable <testname>_<first>, built with every +# member's EIGEN_TEST_PART_<N> defined; unlisted parts stay separate. Only +# parts that differ by CALL_SUBTEST_<N> alone may be grouped, so a source with +# explicit EIGEN_TEST_PART_<N> markers, or a part named individually in the +# smoke-test list, is a configure error. +# # Again, ctest -R allows to run all matching tests. macro(ei_add_test testname) get_property(EIGEN_TESTS_LIST GLOBAL PROPERTY EIGEN_TESTS_LIST) @@ -202,7 +213,42 @@ endif() if( (EIGEN_SPLIT_LARGE_TESTS AND suffixes) OR explicit_suffixes) add_custom_target(${testname}) + set(ei_grouped_suffixes "") + if(EIGEN_SPLIT_LARGE_TESTS AND DEFINED ei_test_part_groups_${testname}) + if(test_source MATCHES "EIGEN_TEST_PART_[0-9]+") + message(FATAL_ERROR "${testname}: parts with EIGEN_TEST_PART_<N> markers cannot be grouped") + endif() + foreach(ei_part_range IN LISTS ei_test_part_groups_${testname}) + if(NOT ei_part_range MATCHES "^([0-9]+)-([0-9]+)$") + message(FATAL_ERROR "${testname}: part group \"${ei_part_range}\" is not of the form <first>-<last>") + endif() + set(ei_part_first ${CMAKE_MATCH_1}) + set(ei_part_last ${CMAKE_MATCH_2}) + set(ei_part_members "") + foreach(suffix ${suffixes}) + if(suffix GREATER_EQUAL ei_part_first AND suffix LESS_EQUAL ei_part_last) + if("${testname}_${suffix}" IN_LIST ei_smoke_test_list) + message(FATAL_ERROR "${testname}_${suffix} is a smoke test and cannot be grouped") + endif() + list(APPEND ei_part_members ${suffix}) + endif() + endforeach() + if(ei_part_members) + list(SORT ei_part_members COMPARE NATURAL) + list(GET ei_part_members 0 ei_part_name) + ei_add_test_internal(${testname} ${testname}_${ei_part_name} "${ARGV1}" "${ARGV2}") + add_dependencies(${testname} ${testname}_${ei_part_name}) + foreach(suffix ${ei_part_members}) + target_compile_definitions(${testname}_${ei_part_name} PRIVATE -DEIGEN_TEST_PART_${suffix}=1) + endforeach() + list(APPEND ei_grouped_suffixes ${ei_part_members}) + endif() + endforeach() + endif() foreach(suffix ${suffixes}) + if(suffix IN_LIST ei_grouped_suffixes) + continue() + endif() ei_add_test_internal(${testname} ${testname}_${suffix} "${ARGV1}" "${ARGV2}") add_dependencies(${testname} ${testname}_${suffix}) target_compile_definitions(${testname}_${suffix} PRIVATE -DEIGEN_TEST_PART_${suffix}=1)
diff --git a/contrib/Eigen/NumericalDiff b/contrib/Eigen/NumericalDiff index 5729082..ac98ce3 100644 --- a/contrib/Eigen/NumericalDiff +++ b/contrib/Eigen/NumericalDiff
@@ -28,13 +28,15 @@ * is a different method and has its own module in Eigen : \ref * AutoDiff_Module. * - * Currently only "Forward" and "Central" schemes are implemented. Those - * are basic methods, and there exist some more elaborated way of - * computing such approximates. They are implemented using both - * proprietary and free software, and usually requires linking to an - * external library. It is very easy for you to write a functor - * using such software, and the purpose is quite orthogonal to what we - * want to achieve with Eigen. + * NumericalDiff implements the "Forward" and "Central" schemes. Stencil + * computes finite-difference weights for arbitrary fixed-size grids and + * derivative orders; the caller supplies the points and chooses the + * step size. The "Forward" and "Central" schemes are basic methods, and + * there exist some more elaborated way of computing such approximates. + * They are implemented using both proprietary and free software, and + * usually requires linking to an external library. It is very easy for + * you to write a functor using such software, and the purpose is quite + * orthogonal to what we want to achieve with Eigen. * * This is why we will not provide wrappers for every great numerical * differentiation software that exist, but should rather stick with those @@ -53,6 +55,7 @@ // IWYU pragma: begin_exports #include "src/NumericalDiff/NumericalDiff.h" +#include "src/NumericalDiff/Stencil.h" // IWYU pragma: end_exports #include "../../Eigen/src/Core/util/ReenableStupidWarnings.h"
diff --git a/contrib/Eigen/StructuredMatrices b/contrib/Eigen/StructuredMatrices index 5ce3a49..70e835e 100644 --- a/contrib/Eigen/StructuredMatrices +++ b/contrib/Eigen/StructuredMatrices
@@ -13,6 +13,7 @@ #include "../../Eigen/Core" #include "../../Eigen/Eigenvalues" #include "../../Eigen/LU" +#include "../../Eigen/QR" #include "../../Eigen/SVD" #include "../../Eigen/SparseLU" #include "../../Eigen/IterativeLinearSolvers" @@ -31,13 +32,14 @@ * - \c Hankel : a (possibly rectangular) Hankel matrix -- a column-reversed * Toeplitz matrix -- with the same O(n log n) products, and square solves * through the Toeplitz equivalent; - * - \c KroneckerOperator : the Kronecker product of two dense, diagonal or - * sparse matrices as an implicit operator; products, solves, least squares, - * eigendecomposition, SVD, inverse and determinant all factor through the - * operands, diagonal (in particular identity) factors are stored and - * applied in diagonal form, sparse factors through sparse-dense products - * and \c SparseLU, and the product materializes into a dense or a sparse - * matrix on assignment; + * - \c KroneckerOperator : the Kronecker product of two dense, diagonal, + * sparse or identity matrices, or of nested Kronecker operators, as an + * implicit operator; products, solves, least squares, eigendecomposition, + * SVD, inverse and determinant all factor through the operands, diagonal + * factors are stored and applied in diagonal form, sparse factors through + * sparse-dense products and \c SparseLU, \c Identity() factors not at all, + * and the product materializes into a dense or a sparse matrix on + * assignment; * - \c DiagonalPlusLowRank : a diagonal matrix plus a rank-k correction, with * O(nk) products and O(nk^2) Woodbury solves, closed under inversion; * - \c Vandermonde : a Vandermonde matrix stored as its nodes, with Horner
diff --git a/contrib/Eigen/src/AutoDiff/AutoDiffScalar.h b/contrib/Eigen/src/AutoDiff/AutoDiffScalar.h index 71405b6..dc54b10 100644 --- a/contrib/Eigen/src/AutoDiff/AutoDiffScalar.h +++ b/contrib/Eigen/src/AutoDiff/AutoDiffScalar.h
@@ -431,6 +431,18 @@ inline typename DerType::Scalar imag(const AutoDiffScalar<DerType>&) { return 0.; } +template <typename DerType> +inline bool(isnan)(const AutoDiffScalar<DerType>& x) { + return (numext::isnan)(x.value()); +} +template <typename DerType> +inline bool(isinf)(const AutoDiffScalar<DerType>& x) { + return (numext::isinf)(x.value()); +} +template <typename DerType> +inline bool(isfinite)(const AutoDiffScalar<DerType>& x) { + return (numext::isfinite)(x.value()); +} template <typename DerType, typename T> inline typename CleanedUpDerType<DerType>::type(min)(const AutoDiffScalar<DerType>& x, const T& y) { typedef typename CleanedUpDerType<DerType>::type ADS;
diff --git a/contrib/Eigen/src/GPU/CuBlasSupport.h b/contrib/Eigen/src/GPU/CuBlasSupport.h index 560e0b2..89241ac 100644 --- a/contrib/Eigen/src/GPU/CuBlasSupport.h +++ b/contrib/Eigen/src/GPU/CuBlasSupport.h
@@ -27,10 +27,25 @@ namespace gpu { namespace internal { -#define EIGEN_CUBLAS_CHECK(expr) \ - do { \ - cublasStatus_t _s = (expr); \ - eigen_assert(_s == CUBLAS_STATUS_SUCCESS && "cuBLAS call failed"); \ +// cublasGetStatusName arrived in cuBLAS 11.6.1 (CUDA 11.4 Update 2); before it, +// a failure is reported by its numeric status. +inline void cublas_check_failed(cublasStatus_t status, const char* expression, const char* file, int line) { +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 110601 + // A user-defined EIGEN_GPU_CHECK_FAILED need not use every argument. + EIGEN_UNUSED_VARIABLE(status); + EIGEN_UNUSED_VARIABLE(expression); + EIGEN_UNUSED_VARIABLE(file); + EIGEN_UNUSED_VARIABLE(line); + EIGEN_GPU_CHECK_FAILED(cublasGetStatusName(status), expression, file, line); +#else + gpu_check_failed_code("cuBLAS", static_cast<int>(status), expression, file, line); +#endif +} + +#define EIGEN_CUBLAS_CHECK(expr) \ + do { \ + const cublasStatus_t _s = (expr); \ + if (_s != CUBLAS_STATUS_SUCCESS) ::Eigen::gpu::internal::cublas_check_failed(_s, #expr, __FILE__, __LINE__); \ } while (0) constexpr cublasOperation_t to_cublas_op(GpuOp op) { @@ -118,11 +133,7 @@ static constexpr cublasComputeType_t value = cuda_compute_type_detail::kDouble; }; -#define EIGEN_CUBLASLT_CHECK(expr) \ - do { \ - cublasStatus_t _s = (expr); \ - eigen_assert(_s == CUBLAS_STATUS_SUCCESS && "cuBLASLt call failed"); \ - } while (0) +#define EIGEN_CUBLASLT_CHECK(expr) EIGEN_CUBLAS_CHECK(expr) // Maximum workspace the heuristic is allowed to consider. This is a preference // ceiling, not an allocation — actual allocation matches the selected algorithm. @@ -132,6 +143,15 @@ #endif static constexpr size_t kCublasLtMaxWorkspaceBytes = EIGEN_CUDA_CUBLASLT_MAX_WORKSPACE_BYTES; +// Workspace each Context gives its cuBLAS handle (cublasSetWorkspace). Without +// one, cuBLAS allocates workspace for its calls itself: a memory node in every +// captured call. 4 MiB matches cuBLAS's default workspace pool before Hopper; +// the cublasSetWorkspace documentation recommends 32 MiB for Hopper. +#ifndef EIGEN_CUDA_CUBLAS_WORKSPACE_BYTES +#define EIGEN_CUDA_CUBLAS_WORKSPACE_BYTES (4 * 1024 * 1024) // 4 MB +#endif +static constexpr size_t kCublasWorkspaceBytes = EIGEN_CUDA_CUBLAS_WORKSPACE_BYTES; + // Algorithm hint for the cublasGemmEx fallback path. constexpr cublasGemmAlgo_t cuda_gemm_algo() { #ifdef EIGEN_NO_CUDA_TENSOR_OPS @@ -302,6 +322,14 @@ entry = plan_cache.insert(key, CublasLtPlanEntry(lt_handle, key, compute, alpha_type, max_workspace_bytes)); } + // cuBLAS reads alpha and beta on the host as its own scalar types. cuComplex + // and cuDoubleComplex declare 8- and 16-byte alignment, which std::complex + // does not guarantee: MSVC aligns std::complex<double> to 8, and a 16-byte + // load through such a pointer faults. Hand the library copies aligned for + // the stricter of the two. + alignas(cuDoubleComplex) const Scalar alpha_val = *alpha; + alignas(cuDoubleComplex) const Scalar beta_val = *beta; + if (entry->use_cublaslt) { const size_t needed = entry->workspace_size; if (needed > workspace.size()) { @@ -310,14 +338,15 @@ workspace = DeviceBuffer(needed); } - EIGEN_CUBLASLT_CHECK(cublasLtMatmul(lt_handle, entry->matmul_desc, alpha, A, entry->layout_A, B, entry->layout_B, - beta, C, entry->layout_C, C, entry->layout_C, &entry->algo, workspace.get(), - needed, stream)); + EIGEN_CUBLASLT_CHECK(cublasLtMatmul(lt_handle, entry->matmul_desc, &alpha_val, A, entry->layout_A, B, + entry->layout_B, &beta_val, C, entry->layout_C, C, entry->layout_C, + &entry->algo, workspace.get(), needed, stream)); } else { // Fallback: cublasGemmEx for shapes/types that cublasLt cannot handle. - EIGEN_CUBLAS_CHECK(EIGEN_CUBLAS_FN(cublasGemmEx)( - cublas_handle, transA, transB, to_blas_dim(m), to_blas_dim(n), to_blas_dim(k), alpha, A, dtype, - to_blas_dim(lda), B, dtype, to_blas_dim(ldb), beta, C, dtype, to_blas_dim(ldc), compute, cuda_gemm_algo())); + EIGEN_CUBLAS_CHECK(EIGEN_CUBLAS_FN(cublasGemmEx)(cublas_handle, transA, transB, to_blas_dim(m), to_blas_dim(n), + to_blas_dim(k), &alpha_val, A, dtype, to_blas_dim(lda), B, dtype, + to_blas_dim(ldb), &beta_val, C, dtype, to_blas_dim(ldc), compute, + cuda_gemm_algo())); } } @@ -339,12 +368,11 @@ static_assert(sizeof(cuDoubleComplex) == sizeof(std::complex<double>), "cuDoubleComplex and std::complex<double> layout mismatch"); -// Complex alpha/beta are type-punned from std::complex<T>* to -// cuComplex*/cuDoubleComplex*. reinterpret_cast violates strict aliasing here: -// once inlined, clang/MSVC no longer see a read through the original type and -// elide the caller's store, which segfaults. std::memcpy is the standard-blessed -// pun. Device array pointers (A, B, C) are never dereferenced by the host -// compiler, so reinterpret_cast is safe for them. +// Complex alpha/beta are copied into cuComplex/cuDoubleComplex locals instead of +// reinterpret_cast: the copy gives cuBLAS the alignment those types declare (see +// cublaslt_gemm) and reads the std::complex through its own type. Device array +// pointers (A, B, C) are never dereferenced by the host compiler, so +// reinterpret_cast is safe for them. inline cublasStatus_t cublasXgemm(cublasHandle_t h, cublasOperation_t transA, cublasOperation_t transB, int64_t m, int64_t n, int64_t k, const std::complex<float>* alpha, const std::complex<float>* A, int64_t lda, const std::complex<float>* B, int64_t ldb, @@ -569,8 +597,35 @@ } // The BLAS-1 wrappers below honour whichever pointer mode the caller set on the -// handle; under CUBLAS_POINTER_MODE_DEVICE the dot/nrm2 result pointers must -// address device memory. +// handle; under CUBLAS_POINTER_MODE_DEVICE the dot/nrm2 result and axpy/scal +// alpha pointers address device memory and are passed through. + +// Complex axpy/scal alpha as cuBLAS reads it. Under CUBLAS_POINTER_MODE_HOST it is +// copied into the cuBLAS type for the alignment reason given in cublaslt_gemm +// (Zaxpy and Zscal fault on a std::complex<double> at 8 mod 16); under +// CUBLAS_POINTER_MODE_DEVICE it addresses device memory and is passed through. +template <typename CuComplexType> +struct Blas1ComplexAlpha { + template <typename Complex> + Blas1ComplexAlpha(cublasHandle_t h, const Complex* alpha) { + static_assert(sizeof(Complex) == sizeof(CuComplexType), "complex alpha layout mismatch"); + cublasPointerMode_t mode = CUBLAS_POINTER_MODE_HOST; + status = cublasGetPointerMode(h, &mode); + if (status != CUBLAS_STATUS_SUCCESS || mode == CUBLAS_POINTER_MODE_DEVICE) { + ptr = reinterpret_cast<const CuComplexType*>(alpha); + } else { + std::memcpy(&value, alpha, sizeof(value)); + ptr = &value; + } + } + // ptr may point at value. + Blas1ComplexAlpha(const Blas1ComplexAlpha&) = delete; + Blas1ComplexAlpha& operator=(const Blas1ComplexAlpha&) = delete; + + CuComplexType value; + const CuComplexType* ptr; + cublasStatus_t status; +}; // dot: result = x^T * y (real) or x^H * y (complex, conjugating x). inline cublasStatus_t cublasXdot(cublasHandle_t h, int64_t n, const float* x, int64_t incx, const float* y, @@ -623,31 +678,32 @@ } inline cublasStatus_t cublasXaxpy(cublasHandle_t h, int64_t n, const std::complex<float>* alpha, const std::complex<float>* x, int64_t incx, std::complex<float>* y, int64_t incy) { - cuComplex a; - std::memcpy(&a, alpha, sizeof(a)); - return EIGEN_CUBLAS_FN(cublasCaxpy)(h, to_blas_dim(n), &a, reinterpret_cast<const cuComplex*>(x), to_blas_dim(incx), - reinterpret_cast<cuComplex*>(y), to_blas_dim(incy)); + const Blas1ComplexAlpha<cuComplex> a(h, alpha); + if (a.status != CUBLAS_STATUS_SUCCESS) return a.status; + return EIGEN_CUBLAS_FN(cublasCaxpy)(h, to_blas_dim(n), a.ptr, reinterpret_cast<const cuComplex*>(x), + to_blas_dim(incx), reinterpret_cast<cuComplex*>(y), to_blas_dim(incy)); } inline cublasStatus_t cublasXaxpy(cublasHandle_t h, int64_t n, const std::complex<double>* alpha, const std::complex<double>* x, int64_t incx, std::complex<double>* y, int64_t incy) { - cuDoubleComplex a; - std::memcpy(&a, alpha, sizeof(a)); - return EIGEN_CUBLAS_FN(cublasZaxpy)(h, to_blas_dim(n), &a, reinterpret_cast<const cuDoubleComplex*>(x), + const Blas1ComplexAlpha<cuDoubleComplex> a(h, alpha); + if (a.status != CUBLAS_STATUS_SUCCESS) return a.status; + return EIGEN_CUBLAS_FN(cublasZaxpy)(h, to_blas_dim(n), a.ptr, reinterpret_cast<const cuDoubleComplex*>(x), to_blas_dim(incx), reinterpret_cast<cuDoubleComplex*>(y), to_blas_dim(incy)); } // SCAL with complex alpha (Cscal/Zscal); the real-alpha forms are above. inline cublasStatus_t cublasXscal(cublasHandle_t h, int64_t n, const std::complex<float>* alpha, std::complex<float>* x, int64_t incx) { - cuComplex a; - std::memcpy(&a, alpha, sizeof(a)); - return EIGEN_CUBLAS_FN(cublasCscal)(h, to_blas_dim(n), &a, reinterpret_cast<cuComplex*>(x), to_blas_dim(incx)); + const Blas1ComplexAlpha<cuComplex> a(h, alpha); + if (a.status != CUBLAS_STATUS_SUCCESS) return a.status; + return EIGEN_CUBLAS_FN(cublasCscal)(h, to_blas_dim(n), a.ptr, reinterpret_cast<cuComplex*>(x), to_blas_dim(incx)); } inline cublasStatus_t cublasXscal(cublasHandle_t h, int64_t n, const std::complex<double>* alpha, std::complex<double>* x, int64_t incx) { - cuDoubleComplex a; - std::memcpy(&a, alpha, sizeof(a)); - return EIGEN_CUBLAS_FN(cublasZscal)(h, to_blas_dim(n), &a, reinterpret_cast<cuDoubleComplex*>(x), to_blas_dim(incx)); + const Blas1ComplexAlpha<cuDoubleComplex> a(h, alpha); + if (a.status != CUBLAS_STATUS_SUCCESS) return a.status; + return EIGEN_CUBLAS_FN(cublasZscal)(h, to_blas_dim(n), a.ptr, reinterpret_cast<cuDoubleComplex*>(x), + to_blas_dim(incx)); } // copy: y = x.
diff --git a/contrib/Eigen/src/GPU/CuDssSupport.h b/contrib/Eigen/src/GPU/CuDssSupport.h index 8664968..a718a8e 100644 --- a/contrib/Eigen/src/GPU/CuDssSupport.h +++ b/contrib/Eigen/src/GPU/CuDssSupport.h
@@ -35,11 +35,11 @@ namespace Eigen { namespace gpu { namespace internal { -#define EIGEN_CUDSS_CHECK(x) \ - do { \ - cudssStatus_t _s = (x); \ - eigen_assert(_s == CUDSS_STATUS_SUCCESS && "cuDSS call failed: " #x); \ - EIGEN_UNUSED_VARIABLE(_s); \ +#define EIGEN_CUDSS_CHECK(x) \ + do { \ + const cudssStatus_t _s = (x); \ + if (_s != CUDSS_STATUS_SUCCESS) \ + ::Eigen::gpu::internal::gpu_check_failed_code("cuDSS", static_cast<int>(_s), #x, __FILE__, __LINE__); \ } while (0) // cuDSS 0.8 changed cudssMatrixCreateDn/Csr from cudaDataType_t to the
diff --git a/contrib/Eigen/src/GPU/CuFftSupport.h b/contrib/Eigen/src/GPU/CuFftSupport.h index 94db97a..10310a8 100644 --- a/contrib/Eigen/src/GPU/CuFftSupport.h +++ b/contrib/Eigen/src/GPU/CuFftSupport.h
@@ -23,11 +23,11 @@ namespace gpu { namespace internal { -#define EIGEN_CUFFT_CHECK(x) \ - do { \ - cufftResult _r = (x); \ - eigen_assert(_r == CUFFT_SUCCESS && "cuFFT call failed: " #x); \ - EIGEN_UNUSED_VARIABLE(_r); \ +#define EIGEN_CUFFT_CHECK(x) \ + do { \ + const cufftResult _r = (x); \ + if (_r != CUFFT_SUCCESS) \ + ::Eigen::gpu::internal::gpu_check_failed_code("cuFFT", static_cast<int>(_r), #x, __FILE__, __LINE__); \ } while (0) template <typename Scalar>
diff --git a/contrib/Eigen/src/GPU/CuSolverSupport.h b/contrib/Eigen/src/GPU/CuSolverSupport.h index 6d4f6f9..08f2055 100644 --- a/contrib/Eigen/src/GPU/CuSolverSupport.h +++ b/contrib/Eigen/src/GPU/CuSolverSupport.h
@@ -19,13 +19,12 @@ #include "./GpuSupport.h" #include <cusolverDn.h> -#include <cstdio> namespace Eigen { namespace gpu { namespace internal { -// cuSOLVER's public API has no cusolverGetErrorString(), so failed asserts would +// cuSOLVER's public API has no cusolverGetErrorString(), so failure reports would // otherwise carry a bare numeric code. inline const char* cusolver_status_name(cusolverStatus_t s) { switch (s) { @@ -52,21 +51,11 @@ } } -inline bool report_cusolver_failure(cusolverStatus_t s, const char* expr, const char* file, int line) { - std::fprintf(stderr, - "cuSOLVER call failed\n" - " expr: %s\n" - " status: %s (%d)\n" - " at: %s:%d\n", - expr, cusolver_status_name(s), static_cast<int>(s), file, line); - return false; -} - -#define EIGEN_CUSOLVER_CHECK(expr) \ - do { \ - cusolverStatus_t _s = (expr); \ - eigen_assert(_s == CUSOLVER_STATUS_SUCCESS || \ - ::Eigen::gpu::internal::report_cusolver_failure(_s, #expr, __FILE__, __LINE__)); \ +#define EIGEN_CUSOLVER_CHECK(expr) \ + do { \ + const cusolverStatus_t _s = (expr); \ + if (_s != CUSOLVER_STATUS_SUCCESS) \ + EIGEN_GPU_CHECK_FAILED(::Eigen::gpu::internal::cusolver_status_name(_s), #expr, __FILE__, __LINE__); \ } while (0) struct CusolverParams {
diff --git a/contrib/Eigen/src/GPU/CuSparseSupport.h b/contrib/Eigen/src/GPU/CuSparseSupport.h index de6e859..f3a9e6a 100644 --- a/contrib/Eigen/src/GPU/CuSparseSupport.h +++ b/contrib/Eigen/src/GPU/CuSparseSupport.h
@@ -37,11 +37,10 @@ namespace gpu { namespace internal { -#define EIGEN_CUSPARSE_CHECK(x) \ - do { \ - cusparseStatus_t _s = (x); \ - eigen_assert(_s == CUSPARSE_STATUS_SUCCESS && "cuSPARSE call failed: " #x); \ - EIGEN_UNUSED_VARIABLE(_s); \ +#define EIGEN_CUSPARSE_CHECK(x) \ + do { \ + const cusparseStatus_t _s = (x); \ + if (_s != CUSPARSE_STATUS_SUCCESS) EIGEN_GPU_CHECK_FAILED(cusparseGetErrorName(_s), #x, __FILE__, __LINE__); \ } while (0) // cuSPARSE rejects CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE for real scalar
diff --git a/contrib/Eigen/src/GPU/DeviceDispatch.h b/contrib/Eigen/src/GPU/DeviceDispatch.h index bd6511e..b8b8da8 100644 --- a/contrib/Eigen/src/GPU/DeviceDispatch.h +++ b/contrib/Eigen/src/GPU/DeviceDispatch.h
@@ -69,6 +69,17 @@ if (resized) { dst.resize(m, n); } + + // cuBLAS rejects ld = 0 (ld >= max(1, rows)) and the cublasLt heuristic + // faults on n == 0, so empty products stop here. For k == 0, C = beta * C + // with beta in {0, 1}; a resized C starts from zero. + if (m == 0 || n == 0 || k == 0) { + if ((resized || beta_val == Scalar(0)) && dst.sizeInBytes() > 0) { + EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(dst.data(), 0, dst.sizeInBytes(), ctx.stream())); + dst.recordReady(ctx.stream()); + } + return; + } const int64_t ldc = dst.rows(); Scalar alpha_local = alpha_scale * traits_lhs::alpha(expr.lhs()) * traits_rhs::alpha(expr.rhs()); @@ -80,13 +91,8 @@ EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(dst.data(), 0, dst.sizeInBytes(), ctx.stream())); } - // cuBLAS reads alpha and beta through host pointers. Holding them in an array - // keeps the compiler from eliding their stack slots — at -O1+ clang and MSVC - // otherwise drop the stores for complex types, leaving cuBLAS with a dangling - // pointer. - Scalar scalars[2] = {alpha_local, beta_val}; - cublaslt_gemm(ctx.cublasLtHandle(), ctx.cublasHandle(), transA, transB, m, n, k, &scalars[0], A.data(), lda, B.data(), - ldb, &scalars[1], dst.data(), ldc, ctx.gemmWorkspace(), ctx.gemmPlanCache(), + cublaslt_gemm(ctx.cublasLtHandle(), ctx.cublasHandle(), transA, transB, m, n, k, &alpha_local, A.data(), lda, + B.data(), ldb, &beta_val, dst.data(), ldc, ctx.gemmWorkspace(), ctx.gemmPlanCache(), ctx.cublasLtMaxWorkspaceBytes(), ctx.stream()); dst.recordReady(ctx.stream()); @@ -291,11 +297,10 @@ dst.resize(m, n); constexpr cublasFillMode_t uplo = (UpLo == Lower) ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER; - // The array keeps the host-pointer stack slots alive; see the GEMM dispatch. - Scalar scalars[2] = {Scalar(1), Scalar(0)}; + const Scalar one(1), zero(0); - EIGEN_CUBLAS_CHECK(cublasXsymm(ctx.cublasHandle(), CUBLAS_SIDE_LEFT, uplo, m, n, &scalars[0], A.data(), A.rows(), - B.data(), B.rows(), &scalars[1], dst.data(), dst.rows())); + EIGEN_CUBLAS_CHECK(cublasXsymm(ctx.cublasHandle(), CUBLAS_SIDE_LEFT, uplo, m, n, &one, A.data(), A.rows(), B.data(), + B.rows(), &zero, dst.data(), dst.rows())); dst.recordReady(ctx.stream()); } @@ -352,10 +357,9 @@ if (m > 0 && n > 0) { A.waitReady(ctx.stream()); B.waitReady(ctx.stream()); - // See the GEMM dispatch: array prevents compiler from eliding host-pointer stack slots. - Scalar scalars[2] = {expr.alpha(), expr.beta()}; - EIGEN_CUBLAS_CHECK(cublasXgeam(ctx.cublasHandle(), CUBLAS_OP_N, CUBLAS_OP_N, m, n, &scalars[0], A.data(), m, - &scalars[1], B.data(), m, dst.data(), m)); + const Scalar alpha_val = expr.alpha(), beta_val = expr.beta(); + EIGEN_CUBLAS_CHECK(cublasXgeam(ctx.cublasHandle(), CUBLAS_OP_N, CUBLAS_OP_N, m, n, &alpha_val, A.data(), m, + &beta_val, B.data(), m, dst.data(), m)); dst.recordReady(ctx.stream()); } } @@ -544,13 +548,24 @@ namespace internal { // Runs `f` with the handle temporarily in CUBLAS_POINTER_MODE_DEVICE, restoring -// the caller's mode afterwards. +// the caller's mode afterwards, also when a failure in `f` throws: the handle +// would otherwise read host scalars of later calls as device pointers. template <typename F> void with_device_pointer_mode(cublasHandle_t h, F&& f) { + struct RestoreOnThrow { + cublasHandle_t handle; + cublasPointerMode_t mode; + bool armed; + ~RestoreOnThrow() { + if (armed) (void)cublasSetPointerMode(handle, mode); // unchecked: may run during unwinding + } + }; cublasPointerMode_t prev; EIGEN_CUBLAS_CHECK(cublasGetPointerMode(h, &prev)); EIGEN_CUBLAS_CHECK(cublasSetPointerMode(h, CUBLAS_POINTER_MODE_DEVICE)); + RestoreOnThrow restore{h, prev, true}; f(); + restore.armed = false; EIGEN_CUBLAS_CHECK(cublasSetPointerMode(h, prev)); } } // namespace internal @@ -564,62 +579,84 @@ } // namespace internal template <typename Scalar_> -DeviceScalar<typename DeviceMatrix<Scalar_>::Scalar> DeviceMatrix<Scalar_>::dot(Context& ctx, - const DeviceMatrix& other) const { +void DeviceMatrix<Scalar_>::dot(Context& ctx, const DeviceMatrix& other, DeviceScalar<Scalar>& result) const { const int64_t n = internal::blas1_size(rows_, cols_); eigen_assert(n == internal::blas1_size(other.rows_, other.cols_)); + eigen_assert(result.stream() == ctx.stream() && "DeviceMatrix::dot: result must live on ctx's stream"); if (n > 0) { - // Allocated uninitialized: cublasXdot overwrites the slot, so uploading a - // zero first would be a wasted H2D transfer per reduction. - DeviceScalar<Scalar> result(ctx.stream()); waitReady(ctx.stream()); other.waitReady(ctx.stream()); internal::with_device_pointer_mode(ctx.cublasHandle(), [&] { EIGEN_CUBLAS_CHECK( internal::cublasXdot(ctx.cublasHandle(), n, data_.get(), 1, other.data_.get(), 1, result.devicePtr())); }); - return result; + } else { + EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(result.devicePtr(), 0, sizeof(Scalar), ctx.stream())); } - return DeviceScalar<Scalar>(Scalar(0), ctx.stream()); } -namespace internal { -// For real Scalar, dot(x,x) already has type DeviceScalar<RealScalar>, so a move -// suffices and nothing syncs. -template <typename Scalar, typename RealScalar> -std::enable_if_t<std::is_same<Scalar, RealScalar>::value, DeviceScalar<RealScalar>> squaredNorm_from_dot( - DeviceScalar<Scalar>&& d, cudaStream_t) { - return std::move(d); +template <typename Scalar_> +DeviceScalar<typename DeviceMatrix<Scalar_>::Scalar> DeviceMatrix<Scalar_>::dot(Context& ctx, + const DeviceMatrix& other) const { + // Allocated uninitialized: the reduction overwrites the slot. + DeviceScalar<Scalar> result(ctx.stream()); + dot(ctx, other, result); + return result; } -// Complex must sync to extract the real part: DeviceScalar arithmetic is real-only. -template <typename Scalar, typename RealScalar> -std::enable_if_t<!std::is_same<Scalar, RealScalar>::value, DeviceScalar<RealScalar>> squaredNorm_from_dot( - DeviceScalar<Scalar>&& d, cudaStream_t stream) { - return DeviceScalar<RealScalar>(numext::real(Scalar(d)), stream); + +template <typename Scalar_> +void DeviceMatrix<Scalar_>::squaredNorm(Context& ctx, DeviceScalar<RealScalar>& result) const { + const int64_t n = internal::blas1_size(rows_, cols_); + eigen_assert(result.stream() == ctx.stream() && "DeviceMatrix::squaredNorm: result must live on ctx's stream"); + if (n > 0) { + // ||x||^2 = x^T x over the 2n real and imaginary parts of a complex x + // (std::complex<T> has the layout of T[2]): real, so no host sync. Unscaled, + // and ~4.5x faster than nrm2^2; stableNorm() is the scaled form. + const int64_t reals = NumTraits<Scalar>::IsComplex ? 2 * n : n; + const RealScalar* x = reinterpret_cast<const RealScalar*>(data_.get()); + waitReady(ctx.stream()); + internal::with_device_pointer_mode(ctx.cublasHandle(), [&] { + EIGEN_CUBLAS_CHECK(internal::cublasXdot(ctx.cublasHandle(), reals, x, 1, x, 1, result.devicePtr())); + }); + } else { + EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(result.devicePtr(), 0, sizeof(RealScalar), ctx.stream())); + } } -} // namespace internal template <typename Scalar_> DeviceScalar<typename NumTraits<Scalar_>::Real> DeviceMatrix<Scalar_>::squaredNorm(Context& ctx) const { - // dot(x,x) rather than nrm2()^2: the dot kernel is ~4.5x faster, since nrm2 - // runs a scaled sum of squares whose overflow protection convergence checks do - // not need. - return internal::squaredNorm_from_dot<Scalar_, RealScalar>(dot(ctx, *this), ctx.stream()); + DeviceScalar<RealScalar> result(ctx.stream()); + squaredNorm(ctx, result); + return result; +} + +template <typename Scalar_> +void DeviceMatrix<Scalar_>::norm(Context& ctx, DeviceScalar<RealScalar>& result) const { + // sqrt of the dot product: a dot and a one-element NPP sqrt cost less than + // cuBLAS nrm2's scaled accumulation, see stableNorm(). + squaredNorm(ctx, result); + internal::device_scalar_sqrt(result.devicePtr(), ctx.nppStreamContext()); } template <typename Scalar_> DeviceScalar<typename NumTraits<Scalar_>::Real> DeviceMatrix<Scalar_>::norm(Context& ctx) const { + DeviceScalar<RealScalar> result(ctx.stream()); + norm(ctx, result); + return result; +} + +template <typename Scalar_> +void DeviceMatrix<Scalar_>::stableNorm(Context& ctx, DeviceScalar<RealScalar>& result) const { const int64_t n = internal::blas1_size(rows_, cols_); + eigen_assert(result.stream() == ctx.stream() && "DeviceMatrix::stableNorm: result must live on ctx's stream"); if (n > 0) { - // See dot(): uninitialized on purpose, cublasXnrm2 overwrites the slot. - DeviceScalar<RealScalar> result(ctx.stream()); waitReady(ctx.stream()); internal::with_device_pointer_mode(ctx.cublasHandle(), [&] { EIGEN_CUBLAS_CHECK(internal::cublasXnrm2(ctx.cublasHandle(), n, data_.get(), 1, result.devicePtr())); }); - return result; + } else { + EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(result.devicePtr(), 0, sizeof(RealScalar), ctx.stream())); } - return DeviceScalar<RealScalar>(RealScalar(0), ctx.stream()); } template <typename Scalar_> @@ -709,6 +746,64 @@ return *this; } +namespace internal { +// x[i] /= alpha. Real Scalar: NPP divides in place. Complex Scalar: cuBLAS scal by the +// host reciprocal, which std::complex computes with scaling, so |alpha| > sqrt(max) +// does not overflow; one extra rounding per element. +inline void divide_in_place(Context& ctx, float* x, int64_t n, float alpha) { + device_divC(alpha, x, Eigen::internal::convert_index<int>(n), ctx.stream()); +} +inline void divide_in_place(Context& ctx, double* x, int64_t n, double alpha) { + device_divC(alpha, x, Eigen::internal::convert_index<int>(n), ctx.stream()); +} +template <typename Real> +void divide_in_place(Context& ctx, std::complex<Real>* x, int64_t n, std::complex<Real> alpha) { + const std::complex<Real> inv = std::complex<Real>(1) / alpha; + EIGEN_CUBLAS_CHECK(cublasXscal(ctx.cublasHandle(), n, &inv, x, 1)); +} +} // namespace internal + +template <typename Scalar_> +void DeviceMatrix<Scalar_>::divide(Context& ctx, Scalar alpha) { + const int64_t n = internal::blas1_size(rows_, cols_); + if (n > 0) { + waitReady(ctx.stream()); + internal::divide_in_place(ctx, data_.get(), n, alpha); + recordReady(ctx.stream()); + } +} + +// this /= alpha +template <typename Scalar_> +DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator/=(Scalar alpha) { + divide(Context::threadLocal(), alpha); + return *this; +} + +// Deep copies: device-to-device cuBLAS copy on the thread-local Context. +template <typename Scalar_> +DeviceMatrix<Scalar_>::DeviceMatrix(const DeviceMatrix& other) : DeviceMatrix() { + copyFrom(Context::threadLocal(), other); +} + +template <typename Scalar_> +DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const DeviceMatrix& other) { + if (this != &other) copyFrom(Context::threadLocal(), other); + return *this; +} + +template <typename Scalar_> +DeviceScalar<typename NumTraits<Scalar_>::Real> DeviceMatrix<Scalar_>::stableNorm(Context& ctx) const { + DeviceScalar<RealScalar> result(ctx.stream()); + stableNorm(ctx, result); + return result; +} + +template <typename Scalar_> +DeviceScalar<typename NumTraits<Scalar_>::Real> DeviceMatrix<Scalar_>::stableNorm() const { + return stableNorm(Context::threadLocal()); +} + // this *= alpha (scal, device pointer — avoids host sync) template <typename Scalar_> DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator*=(const DeviceScalar<Scalar>& alpha) {
diff --git a/contrib/Eigen/src/GPU/DeviceExpr.h b/contrib/Eigen/src/GPU/DeviceExpr.h index f0880f4..bc806a7 100644 --- a/contrib/Eigen/src/GPU/DeviceExpr.h +++ b/contrib/Eigen/src/GPU/DeviceExpr.h
@@ -18,7 +18,7 @@ // AdjointView<S> — d_A.adjoint() → marks ConjTrans for GEMM // TransposeView<S> — d_A.transpose() → marks Trans for GEMM // Scaled<Expr> — alpha * expr → carries scalar factor -// gpu::GemmExpr<Lhs, Rhs> — lhs * rhs → dispatches to cublasXgemm +// gpu::GemmExpr<Lhs, Rhs> — lhs * rhs → dispatches to cublasLtMatmul (cublasGemmEx fallback) #ifndef EIGEN_GPU_DEVICE_EXPR_H #define EIGEN_GPU_DEVICE_EXPR_H
diff --git a/contrib/Eigen/src/GPU/DeviceMatrix.h b/contrib/Eigen/src/GPU/DeviceMatrix.h index 2b57c78..a2290b4 100644 --- a/contrib/Eigen/src/GPU/DeviceMatrix.h +++ b/contrib/Eigen/src/GPU/DeviceMatrix.h
@@ -58,9 +58,10 @@ /** Non-blocking check: has the transfer completed? */ bool ready() const { if (synced_) return true; - cudaError_t err = cudaEventQuery(event_); + const cudaError_t err = cudaEventQuery(event_); if (err == cudaSuccess) return true; - eigen_assert(err == cudaErrorNotReady && "cudaEventQuery failed"); + if (err != cudaErrorNotReady) + EIGEN_GPU_CHECK_FAILED(cudaGetErrorName(err), "cudaEventQuery(event_)", __FILE__, __LINE__); return false; } @@ -158,6 +159,7 @@ template <int UpLo> DeviceMatrix(const SymmExpr<Scalar, UpLo>& expr); DeviceMatrix(const SpMVExpr<Scalar>& expr); + DeviceMatrix(const SpMVAffineExpr<Scalar>& expr); ~DeviceMatrix() { // cudaEventDestroy on a pending event is non-blocking: the runtime defers @@ -202,8 +204,14 @@ return *this; } - DeviceMatrix(const DeviceMatrix&) = delete; - DeviceMatrix& operator=(const DeviceMatrix&) = delete; + /** Deep copy: a device-to-device cuBLAS copy on the thread-local Context, + * asynchronous and without a host transfer. Copies exist so that generic Eigen + * algorithm code with value semantics (`p = precond.solve(residual)` in + * internal::conjugate_gradient) compiles against DeviceMatrix; code that + * manages streams explicitly should prefer copyFrom(ctx, other). Defined + * out-of-line in DeviceDispatch.h, where Context is complete. */ + DeviceMatrix(const DeviceMatrix& other); + DeviceMatrix& operator=(const DeviceMatrix& other); /** Upload a host Eigen matrix to device memory (synchronous). * @@ -342,6 +350,12 @@ Index cols() const { return cols_; } bool empty() const { return rows_ == 0 || cols_ == 0; } + /** Whether this matrix holds storage borrowed through view(). Destroying a + * view does not free that storage, and release() on a view returns a pointer + * that its owner still frees. resize() to a different size replaces a view + * with storage of its own; resize() to the same size leaves it a view. */ + bool isView() const { return data_ != nullptr && data_.get_deleter().borrow; } + /** Size of the device allocation in bytes. */ size_t sizeInBytes() const { return static_cast<size_t>(rows_) * static_cast<size_t>(cols_) * sizeof(Scalar); } @@ -431,13 +445,27 @@ * DeviceScalar's conversion to Scalar, which syncs. */ DeviceScalar<Scalar> dot(Context& ctx, const DeviceMatrix& other) const; - /** Squared L2 norm via dot(x, x). For real types the result stays on device; - * for complex it syncs, since DeviceScalar arithmetic is real-only. */ + /** Squared L2 norm via dot(x, x), without a host sync. */ DeviceScalar<typename NumTraits<Scalar>::Real> squaredNorm(Context& ctx) const; - /** L2 norm, without a host sync. */ + /** L2 norm as sqrt(squaredNorm()), like MatrixBase::norm(), without a host + * sync: unscaled, so it overflows once some |x_i| > sqrt(max) and loses + * accuracy when every |x_i| < sqrt(min). */ DeviceScalar<typename NumTraits<Scalar>::Real> norm(Context& ctx) const; + /** Overflow-safe L2 norm, like MatrixBase::stableNorm(): cuBLAS nrm2's scaled + * sum of squares, without a host sync. Eigen's iterative solver templates call it. */ + DeviceScalar<typename NumTraits<Scalar>::Real> stableNorm(Context& ctx) const; + + /** The reductions above written into \p result's existing storage on \p ctx, + * instead of a new DeviceScalar: no allocation per call, so a loop or a + * captured CUDA graph repeats them without allocator traffic. \p result must + * live on ctx.stream(). */ + void dot(Context& ctx, const DeviceMatrix& other, DeviceScalar<Scalar>& result) const; + void squaredNorm(Context& ctx, DeviceScalar<RealScalar>& result) const; + void norm(Context& ctx, DeviceScalar<RealScalar>& result) const; + void stableNorm(Context& ctx, DeviceScalar<RealScalar>& result) const; + /** Set all elements to zero. */ void setZero(Context& ctx); void setZero(cudaStream_t stream); @@ -448,12 +476,18 @@ /** this *= alpha (cuBLAS scal). */ void scale(Context& ctx, Scalar alpha); + /** this /= alpha. A true division for real Scalar (NPP divide-by-constant: one + * rounding per element, and no overflow of 1/alpha for subnormal alpha); complex + * Scalar scales by the host reciprocal, one extra rounding. */ + void divide(Context& ctx, Scalar alpha); + /** Deep copy: this = other (cuBLAS copy). Resizes if needed. */ void copyFrom(Context& ctx, const DeviceMatrix& other); DeviceScalar<Scalar> dot(const DeviceMatrix& other) const; DeviceScalar<typename NumTraits<Scalar>::Real> squaredNorm() const; DeviceScalar<typename NumTraits<Scalar>::Real> norm() const; + DeviceScalar<typename NumTraits<Scalar>::Real> stableNorm() const; void setZero(); // The operators below let iterative-solver code written against Matrix — say @@ -476,6 +510,9 @@ /** this *= alpha (cuBLAS scal, host pointer mode). */ DeviceMatrix& operator*=(Scalar alpha); + /** this /= alpha, see divide(). */ + DeviceMatrix& operator/=(Scalar alpha); + /** this *= alpha (cuBLAS scal, device pointer mode). Avoids a host sync. */ DeviceMatrix& operator*=(const DeviceScalar<Scalar>& alpha); @@ -492,9 +529,15 @@ /** this -= DeviceScalar * x (cuBLAS axpy with negated device scalar). */ DeviceMatrix& operator-=(const DeviceScaledDevice<Scalar>& expr); - /** Assign from an SpMV expression: d_y = d_A * d_x. */ + /** Assign from an SpMV expression: d_y = d_A * d_x (one cuSPARSE call). */ DeviceMatrix& operator=(const SpMVExpr<Scalar>& expr); + /** Assign from an SpMV expression with a dense addend: d_y = d_b - d_A * d_x and + * the other sign combinations. Copies the addend into d_y (skipped when it is + * d_y), then one cuSPARSE call with beta = ±1. The addend must have the shape + * of the product, and d_x must not be d_y. */ + DeviceMatrix& operator=(const SpMVAffineExpr<Scalar>& expr); + /** Assign from an add expression: d_C = alpha * d_A + beta * d_B (cuBLAS geam). */ DeviceMatrix& operator=(const DeviceAddExpr<Scalar>& expr); @@ -539,7 +582,9 @@ return dm; } - /** Transfer ownership of the device pointer out. Zeros internal state. */ + /** Give up the device pointer and zero the internal state. An owning matrix + * transfers ownership to the caller; a view (isView()) returns its borrowed + * pointer, which its owner still frees. */ Scalar* release() { Scalar* p = data_.release(); rows_ = 0;
diff --git a/contrib/Eigen/src/GPU/DeviceScalar.h b/contrib/Eigen/src/GPU/DeviceScalar.h index e24d122..54802f1 100644 --- a/contrib/Eigen/src/GPU/DeviceScalar.h +++ b/contrib/Eigen/src/GPU/DeviceScalar.h
@@ -53,8 +53,22 @@ return *this; } - DeviceScalar(const DeviceScalar&) = delete; - DeviceScalar& operator=(const DeviceScalar&) = delete; + /** Deep copy: a device-to-device cudaMemcpyAsync on the source's stream, no + * host round trip. Provided so that generic code returning a DeviceScalar by + * value from a const reference (numext::real in Eigen's iterative solver + * templates) compiles; explicit code should move instead. */ + DeviceScalar(const DeviceScalar& o) : d_val_(sizeof(Scalar)), stream_(o.stream_) { + EIGEN_CUDA_RUNTIME_CHECK( + cudaMemcpyAsync(d_val_.get(), o.d_val_.get(), sizeof(Scalar), cudaMemcpyDeviceToDevice, stream_)); + } + + /** Copy assignment adopts the source's stream. Copy construction followed by a + * move releases the previous buffer instead of writing into it: a write on the + * source's stream could race with reads still queued on this scalar's old stream. */ + DeviceScalar& operator=(const DeviceScalar& o) { + if (this != &o) *this = DeviceScalar(o); + return *this; + } /** Download from device, synchronizing the stream. */ Scalar get() const { @@ -73,9 +87,10 @@ cudaStream_t stream() const { return stream_; } // The arithmetic below keeps results on device via the NPP helpers in - // DeviceScalarOps.h, and covers real types only; complex division falls back to - // the implicit conversion and its host sync. Unlike DeviceMatrix, DeviceScalar - // tracks no cross-stream readiness, so all operands must share one stream. + // DeviceScalarOps.h, which cover real types only: dividing or negating a + // complex DeviceScalar does not compile, so convert it to Scalar (a host sync) + // first. Unlike DeviceMatrix, DeviceScalar tracks no cross-stream readiness, + // so all operands must share one stream. friend DeviceScalar operator/(const DeviceScalar& a, const DeviceScalar& b) { eigen_assert(a.stream_ == b.stream_ && "DeviceScalar operator/: operands must share the same stream");
diff --git a/contrib/Eigen/src/GPU/DeviceScalarOps.h b/contrib/Eigen/src/GPU/DeviceScalarOps.h index fb003f8..fe78292 100644 --- a/contrib/Eigen/src/GPU/DeviceScalarOps.h +++ b/contrib/Eigen/src/GPU/DeviceScalarOps.h
@@ -24,6 +24,14 @@ namespace gpu { namespace internal { +// NPP statuses below NPP_SUCCESS are errors; positive ones are warnings. +#define EIGEN_NPP_CHECK(expr) \ + do { \ + const NppStatus _s = (expr); \ + if (_s < NPP_SUCCESS) \ + ::Eigen::gpu::internal::gpu_check_failed_code("NPP", static_cast<int>(_s), #expr, __FILE__, __LINE__); \ + } while (0) + // NPP requires nCudaDeviceId and the device attributes to match the device that // owns `stream`. Re-querying per call is cheap next to the NPP launch itself and // keeps multi-device and borrowed-stream callers correct. @@ -61,45 +69,65 @@ // pDst[i] = pSrc2[i] / pSrc1[i]. inline void device_scalar_div(const float* a, const float* b, float* c, cudaStream_t stream) { NppStreamContext npp_ctx = make_npp_stream_ctx(stream); - nppsDiv_32f_Ctx(b, a, c, 1, npp_ctx); + EIGEN_NPP_CHECK(nppsDiv_32f_Ctx(b, a, c, 1, npp_ctx)); } inline void device_scalar_div(const double* a, const double* b, double* c, cudaStream_t stream) { NppStreamContext npp_ctx = make_npp_stream_ctx(stream); - nppsDiv_64f_Ctx(b, a, c, 1, npp_ctx); + EIGEN_NPP_CHECK(nppsDiv_64f_Ctx(b, a, c, 1, npp_ctx)); +} + +// a = sqrt(a), in place, with a context filled in before any capture began. +inline void device_scalar_sqrt(float* a, const NppStreamContext& npp_ctx) { + EIGEN_NPP_CHECK(nppsSqrt_32f_I_Ctx(a, 1, npp_ctx)); +} + +inline void device_scalar_sqrt(double* a, const NppStreamContext& npp_ctx) { + EIGEN_NPP_CHECK(nppsSqrt_64f_I_Ctx(a, 1, npp_ctx)); } // c = -a. inline void device_scalar_neg(const float* a, float* c, cudaStream_t stream) { NppStreamContext npp_ctx = make_npp_stream_ctx(stream); - nppsMulC_32f_Ctx(a, -1.0f, c, 1, npp_ctx); + EIGEN_NPP_CHECK(nppsMulC_32f_Ctx(a, -1.0f, c, 1, npp_ctx)); } inline void device_scalar_neg(const double* a, double* c, cudaStream_t stream) { NppStreamContext npp_ctx = make_npp_stream_ctx(stream); - nppsMulC_64f_Ctx(a, -1.0, c, 1, npp_ctx); + EIGEN_NPP_CHECK(nppsMulC_64f_Ctx(a, -1.0, c, 1, npp_ctx)); } // c[i] = a[i] * b[i]. inline void device_cwiseProduct(const float* a, const float* b, float* c, int n, cudaStream_t stream) { NppStreamContext npp_ctx = make_npp_stream_ctx(stream); - nppsMul_32f_Ctx(a, b, c, n, npp_ctx); + EIGEN_NPP_CHECK(nppsMul_32f_Ctx(a, b, c, n, npp_ctx)); } inline void device_cwiseProduct(const double* a, const double* b, double* c, int n, cudaStream_t stream) { NppStreamContext npp_ctx = make_npp_stream_ctx(stream); - nppsMul_64f_Ctx(a, b, c, n, npp_ctx); + EIGEN_NPP_CHECK(nppsMul_64f_Ctx(a, b, c, n, npp_ctx)); } // c[i] = a[i] / b[i], with operands swapped as in device_scalar_div. inline void device_cwiseQuotient(const float* a, const float* b, float* c, int n, cudaStream_t stream) { NppStreamContext npp_ctx = make_npp_stream_ctx(stream); - nppsDiv_32f_Ctx(b, a, c, n, npp_ctx); + EIGEN_NPP_CHECK(nppsDiv_32f_Ctx(b, a, c, n, npp_ctx)); } inline void device_cwiseQuotient(const double* a, const double* b, double* c, int n, cudaStream_t stream) { NppStreamContext npp_ctx = make_npp_stream_ctx(stream); - nppsDiv_64f_Ctx(b, a, c, n, npp_ctx); + EIGEN_NPP_CHECK(nppsDiv_64f_Ctx(b, a, c, n, npp_ctx)); +} + +// x[i] /= alpha, a true division (NPP divide-by-constant, in place). +inline void device_divC(float alpha, float* x, int n, cudaStream_t stream) { + NppStreamContext npp_ctx = make_npp_stream_ctx(stream); + EIGEN_NPP_CHECK(nppsDivC_32f_I_Ctx(alpha, x, n, npp_ctx)); +} + +inline void device_divC(double alpha, double* x, int n, cudaStream_t stream) { + NppStreamContext npp_ctx = make_npp_stream_ctx(stream); + EIGEN_NPP_CHECK(nppsDivC_64f_I_Ctx(alpha, x, n, npp_ctx)); } } // namespace internal
diff --git a/contrib/Eigen/src/GPU/FwdDecl.h b/contrib/Eigen/src/GPU/FwdDecl.h index 9cf0457..714b917 100644 --- a/contrib/Eigen/src/GPU/FwdDecl.h +++ b/contrib/Eigen/src/GPU/FwdDecl.h
@@ -89,6 +89,8 @@ class DeviceSparseView; template <typename Scalar_> class SpMVExpr; +template <typename Scalar_> +class SpMVAffineExpr; } // namespace gpu } // namespace Eigen
diff --git a/contrib/Eigen/src/GPU/GpuContext.h b/contrib/Eigen/src/GPU/GpuContext.h index fed40e4..8ebb5ce 100644 --- a/contrib/Eigen/src/GPU/GpuContext.h +++ b/contrib/Eigen/src/GPU/GpuContext.h
@@ -17,9 +17,10 @@ // IWYU pragma: private #include "./InternalHeaderCheck.h" +#include "./DeviceScalarOps.h" #include "./CuBlasSupport.h" #include "./CuSolverSupport.h" -#include <cusparse.h> +#include "./CuSparseSupport.h" #include <vector> namespace Eigen { @@ -84,12 +85,14 @@ cudaStream_t s = nullptr; EIGEN_CUDA_RUNTIME_CHECK(cudaStreamCreate(&s)); stream_ = internal::UniqueStream(s); + npp_stream_ctx_ = internal::make_npp_stream_ctx(stream_.get()); init_cublas(); } /** Create a context on an existing stream (e.g., stream 0 = nullptr). * The caller retains ownership of the stream — this context will not destroy it. */ explicit Context(cudaStream_t stream) : stream_(stream, internal::CudaStreamDeleter{/*owns=*/false}) { + npp_stream_ctx_ = internal::make_npp_stream_ctx(stream_.get()); init_cublas(); } @@ -130,6 +133,11 @@ cudaStream_t stream() const { return stream_.get(); } cublasHandle_t cublasHandle() const { return cublas_.get(); } + /** NPP stream context for stream(), filled in at construction. Filling it + * queries the stream, which fails while the stream is being captured on some + * drivers, so NPP calls inside a capture must use this one. */ + const NppStreamContext& nppStreamContext() const { return npp_stream_ctx_; } + /** Returns the cuSOLVER handle, creating it on first call. */ cusolverDnHandle_t cusolverHandle() { if (!cusolver_) { @@ -179,13 +187,9 @@ cusparseHandle_t cusparseHandle() { if (!cusparse_) { cusparseHandle_t h = nullptr; - cusparseStatus_t s1 = cusparseCreate(&h); - eigen_assert(s1 == CUSPARSE_STATUS_SUCCESS && "cusparseCreate failed"); - EIGEN_UNUSED_VARIABLE(s1); + EIGEN_CUSPARSE_CHECK(cusparseCreate(&h)); cusparse_ = LazyCusparseHandle(h, &destroyCusparse); - cusparseStatus_t s2 = cusparseSetStream(h, stream_.get()); - eigen_assert(s2 == CUSPARSE_STATUS_SUCCESS && "cusparseSetStream failed"); - EIGEN_UNUSED_VARIABLE(s2); + EIGEN_CUSPARSE_CHECK(cusparseSetStream(h, stream_.get())); } return cusparse_.get(); } @@ -202,7 +206,9 @@ // Destroyed in reverse declaration order: the plan cache before the cuBLASLt handle, the stream last. internal::UniqueStream stream_; + NppStreamContext npp_stream_ctx_ = {}; internal::UniqueCublasHandle cublas_; + internal::DeviceBuffer cublas_workspace_; // cublasSetWorkspace; freed before the handle LazyCusolverHandle cusolver_{nullptr, nullptr}; LazyCusparseHandle cusparse_{nullptr, nullptr}; internal::UniqueCublasLtHandle cublas_lt_; // lazy @@ -221,6 +227,8 @@ EIGEN_CUBLAS_CHECK(cublasCreate(&h)); cublas_ = internal::UniqueCublasHandle(h); EIGEN_CUBLAS_CHECK(cublasSetStream(h, stream_.get())); + cublas_workspace_ = internal::DeviceBuffer(internal::kCublasWorkspaceBytes); + EIGEN_CUBLAS_CHECK(cublasSetWorkspace(h, cublas_workspace_.get(), cublas_workspace_.size())); } };
diff --git a/contrib/Eigen/src/GPU/GpuEigenSolver.h b/contrib/Eigen/src/GPU/GpuEigenSolver.h index 0198230..5429417 100644 --- a/contrib/Eigen/src/GPU/GpuEigenSolver.h +++ b/contrib/Eigen/src/GPU/GpuEigenSolver.h
@@ -46,7 +46,7 @@ compute(d_A, options); } - /** Decompose a device-resident A immediately (adopt, no copy). */ + /** Decompose a device-resident A immediately (adopt, no copy; a view is copied). */ explicit SelfAdjointEigenSolver(DeviceMatrix<Scalar>&& d_A, int options = ComputeEigenvectors) { compute(std::move(d_A), options); } @@ -64,6 +64,12 @@ compute(d_A, options); } + /** Bind to \p ctx and decompose a device-resident A (adopt, no copy; a view is copied). */ + SelfAdjointEigenSolver(Context& ctx, DeviceMatrix<Scalar>&& d_A, int options = ComputeEigenvectors) + : solver_ctx_(ctx) { + compute(std::move(d_A), options); + } + ~SelfAdjointEigenSolver() = default; SelfAdjointEigenSolver(const SelfAdjointEigenSolver&) = delete; @@ -116,8 +122,10 @@ } /** Decompose a device matrix (move): the buffer is adopted and overwritten - * in place by syevd — no copy. */ + * in place by syevd — no copy. A view is copied instead: its storage belongs + * to another object. */ SelfAdjointEigenSolver& compute(DeviceMatrix<Scalar>&& d_A, int options = ComputeEigenvectors) { + if (d_A.isView()) return compute(static_cast<const DeviceMatrix<Scalar>&>(d_A), options); if (!begin_compute(d_A, options)) return *this; d_A_ = internal::DeviceBuffer::adopt(static_cast<void*>(d_A.release()), @@ -137,8 +145,7 @@ eigen_assert(solver_ctx_.info() == Success); RealVector W(n_); if (n_ > 0) { - EIGEN_CUDA_RUNTIME_CHECK( - cudaMemcpy(W.data(), d_W_.get(), static_cast<size_t>(n_) * sizeof(RealScalar), cudaMemcpyDeviceToHost)); + solver_ctx_.download(W.data(), d_W_.get(), static_cast<size_t>(n_) * sizeof(RealScalar)); } return W; } @@ -150,9 +157,7 @@ eigen_assert(compute_eigenvectors_ && "eigenvectors() requires ComputeEigenvectors option"); PlainMatrix V(n_, n_); if (n_ > 0) { - EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy(V.data(), d_A_.get(), - static_cast<size_t>(lda_) * static_cast<size_t>(n_) * sizeof(Scalar), - cudaMemcpyDeviceToHost)); + solver_ctx_.download(V.data(), d_A_.get(), static_cast<size_t>(lda_) * static_cast<size_t>(n_) * sizeof(Scalar)); } return V; } @@ -186,7 +191,9 @@ private: mutable internal::GpuSolverContext solver_ctx_; - internal::DeviceBuffer d_A_; // grow-only; overwritten with eigenvectors by syevd + // Overwritten with eigenvectors by syevd. Host and rvalue input are adopted; + // the const& path is grow-only. + internal::DeviceBuffer d_A_; internal::DeviceBuffer d_W_; // grow-only; eigenvalues (RealScalar, length n) bool compute_eigenvectors_ = true; int64_t n_ = 0;
diff --git a/contrib/Eigen/src/GPU/GpuLLT.h b/contrib/Eigen/src/GPU/GpuLLT.h index f72eab6..76ada76 100644 --- a/contrib/Eigen/src/GPU/GpuLLT.h +++ b/contrib/Eigen/src/GPU/GpuLLT.h
@@ -63,7 +63,7 @@ /** Factor a device-resident A immediately (D2D copy). */ explicit LLT(const DeviceMatrix<Scalar>& d_A) { compute(d_A); } - /** Factor a device-resident A immediately (adopt, no copy). */ + /** Factor a device-resident A immediately (adopt, no copy; a view is copied). */ explicit LLT(DeviceMatrix<Scalar>&& d_A) { compute(std::move(d_A)); } /** Bind to \p ctx and factor A immediately. */ @@ -75,6 +75,9 @@ /** Bind to \p ctx and factor a device-resident A (D2D copy). */ LLT(Context& ctx, const DeviceMatrix<Scalar>& d_A) : solver_ctx_(ctx) { compute(d_A); } + /** Bind to \p ctx and factor a device-resident A (adopt, no copy; a view is copied). */ + LLT(Context& ctx, DeviceMatrix<Scalar>&& d_A) : solver_ctx_(ctx) { compute(std::move(d_A)); } + ~LLT() = default; // Non-copyable (owns device memory and library handles). @@ -135,8 +138,10 @@ return *this; } - /** Compute the Cholesky factorization from a device matrix (move, no copy). */ + /** Compute the Cholesky factorization from a device matrix (move, no copy). + * A view is copied instead: its storage belongs to another object. */ LLT& compute(DeviceMatrix<Scalar>&& d_A) { + if (d_A.isView()) return compute(static_cast<const DeviceMatrix<Scalar>&>(d_A)); eigen_assert(d_A.rows() == d_A.cols()); if (!begin_compute(d_A.rows())) return *this; @@ -195,8 +200,10 @@ } /** Solve in place: consumes \p d_B and returns it holding the solution — - * no RHS copy and no allocation (potrs overwrites its RHS). */ + * no RHS copy and no allocation (potrs overwrites its RHS). A view is copied + * instead: its storage belongs to another object. */ DeviceMatrix<Scalar> solve(DeviceMatrix<Scalar>&& d_B) const { + if (d_B.isView()) return solve(static_cast<const DeviceMatrix<Scalar>&>(d_B)); eigen_assert(solver_ctx_.info() == Success && "LLT::solve called on a failed or uninitialized factorization"); eigen_assert(d_B.rows() == n_); d_B.waitReady(solver_ctx_.stream()); @@ -213,7 +220,7 @@ private: mutable internal::GpuSolverContext solver_ctx_; - internal::DeviceBuffer d_factor_; // grow-only + internal::DeviceBuffer d_factor_; // adopted from an rvalue input, else grow-only int64_t n_ = 0; int64_t lda_ = 0;
diff --git a/contrib/Eigen/src/GPU/GpuLU.h b/contrib/Eigen/src/GPU/GpuLU.h index 93dd98a..b97c43d 100644 --- a/contrib/Eigen/src/GPU/GpuLU.h +++ b/contrib/Eigen/src/GPU/GpuLU.h
@@ -55,7 +55,7 @@ /** Factor a device-resident A immediately (D2D copy). */ explicit LU(const DeviceMatrix<Scalar>& d_A) { compute(d_A); } - /** Factor a device-resident A immediately (adopt, no copy). */ + /** Factor a device-resident A immediately (adopt, no copy; a view is copied). */ explicit LU(DeviceMatrix<Scalar>&& d_A) { compute(std::move(d_A)); } /** Bind to \p ctx and factor A immediately. */ @@ -67,6 +67,9 @@ /** Bind to \p ctx and factor a device-resident A (D2D copy). */ LU(Context& ctx, const DeviceMatrix<Scalar>& d_A) : solver_ctx_(ctx) { compute(d_A); } + /** Bind to \p ctx and factor a device-resident A (adopt, no copy; a view is copied). */ + LU(Context& ctx, DeviceMatrix<Scalar>&& d_A) : solver_ctx_(ctx) { compute(std::move(d_A)); } + ~LU() = default; LU(const LU&) = delete; @@ -130,8 +133,10 @@ return *this; } - /** Compute the LU factorization from a device matrix (move, no copy). */ + /** Compute the LU factorization from a device matrix (move, no copy). A view + * is copied instead: its storage belongs to another object. */ LU& compute(DeviceMatrix<Scalar>&& d_A) { + if (d_A.isView()) return compute(static_cast<const DeviceMatrix<Scalar>&>(d_A)); eigen_assert(d_A.rows() == d_A.cols() && "LU requires a square matrix"); if (!begin_compute(d_A.rows())) return *this; @@ -194,8 +199,10 @@ } /** Solve in place: consumes \p d_B and returns it holding the solution — - * no RHS copy and no allocation (getrs overwrites its RHS). */ + * no RHS copy and no allocation (getrs overwrites its RHS). A view is copied + * instead: its storage belongs to another object. */ DeviceMatrix<Scalar> solve(DeviceMatrix<Scalar>&& d_B, GpuOp op = GpuOp::NoTrans) const { + if (d_B.isView()) return solve(static_cast<const DeviceMatrix<Scalar>&>(d_B), op); eigen_assert(solver_ctx_.info() == Success && "LU::solve called on a failed or uninitialized factorization"); eigen_assert(d_B.rows() == n_); d_B.waitReady(solver_ctx_.stream()); @@ -213,7 +220,7 @@ private: mutable internal::GpuSolverContext solver_ctx_; - internal::DeviceBuffer d_lu_; // grow-only + internal::DeviceBuffer d_lu_; // adopted from an rvalue input, else grow-only internal::DeviceBuffer d_ipiv_; // grow-only int64_t n_ = 0; int64_t lda_ = 0;
diff --git a/contrib/Eigen/src/GPU/GpuQR.h b/contrib/Eigen/src/GPU/GpuQR.h index c22a9cb..02a2dc4 100644 --- a/contrib/Eigen/src/GPU/GpuQR.h +++ b/contrib/Eigen/src/GPU/GpuQR.h
@@ -47,7 +47,7 @@ explicit QR(const DeviceMatrix<Scalar>& d_A) { compute(d_A); } - /** Factor a device-resident A immediately (adopt when m >= n, no copy). */ + /** Factor a device-resident A immediately (adopt when m >= n, no copy; a view is copied). */ explicit QR(DeviceMatrix<Scalar>&& d_A) { compute(std::move(d_A)); } /** Bind to \p ctx and factor A immediately. */ @@ -59,6 +59,9 @@ /** Bind to \p ctx and factor a device-resident A (D2D copy). */ QR(Context& ctx, const DeviceMatrix<Scalar>& d_A) : solver_ctx_(ctx) { compute(d_A); } + /** Bind to \p ctx and factor a device-resident A (adopt when m >= n, no copy; a view is copied). */ + QR(Context& ctx, DeviceMatrix<Scalar>&& d_A) : solver_ctx_(ctx) { compute(std::move(d_A)); } + ~QR() = default; QR(const QR&) = delete; @@ -121,8 +124,10 @@ } /** Factor a device matrix (move). For m >= n the buffer is adopted and - * factored in place — no copy; for m < n a transposed copy is unavoidable. */ + * factored in place — no copy; for m < n a transposed copy is unavoidable. A + * view is copied: its storage belongs to another object. */ QR& compute(DeviceMatrix<Scalar>&& d_A) { + if (d_A.isView()) return compute(static_cast<const DeviceMatrix<Scalar>&>(d_A)); if (!begin_compute(d_A)) return *this; if (transposed_) { @@ -180,9 +185,8 @@ eigen_assert(!transposed_ && "matrixR() not available when m < n (we factored A^H internally)"); PlainMatrix qr_full(m_, n_); if (m_ > 0 && n_ > 0) { - EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy(qr_full.data(), d_qr_.get(), - static_cast<size_t>(lda_) * static_cast<size_t>(n_) * sizeof(Scalar), - cudaMemcpyDeviceToHost)); + solver_ctx_.download(qr_full.data(), d_qr_.get(), + static_cast<size_t>(lda_) * static_cast<size_t>(n_) * sizeof(Scalar)); } PlainMatrix R = qr_full.topRows(k()).template triangularView<Upper>(); return R; @@ -190,7 +194,9 @@ private: mutable internal::GpuSolverContext solver_ctx_; - internal::DeviceBuffer d_qr_; // grow-only; QR factors (reflectors below diag, R above) + // QR factors (reflectors below diag, R above). Host and rvalue input are + // adopted when m >= n; the copying paths are grow-only. + internal::DeviceBuffer d_qr_; internal::DeviceBuffer d_tau_; // grow-only; Householder scalars (length k) int64_t m_ = 0; // original A.rows() int64_t n_ = 0; // original A.cols()
diff --git a/contrib/Eigen/src/GPU/GpuSVD.h b/contrib/Eigen/src/GPU/GpuSVD.h index 600a873..98f9c61 100644 --- a/contrib/Eigen/src/GPU/GpuSVD.h +++ b/contrib/Eigen/src/GPU/GpuSVD.h
@@ -49,7 +49,7 @@ compute(d_A, options); } - /** Decompose a device-resident A immediately (adopt when m >= n, no copy). */ + /** Decompose a device-resident A immediately (adopt when m >= n, no copy; a view is copied). */ explicit SVD(DeviceMatrix<Scalar>&& d_A, unsigned int options = ComputeThinU | ComputeThinV) { compute(std::move(d_A), options); } @@ -67,6 +67,11 @@ compute(d_A, options); } + /** Bind to \p ctx and decompose a device-resident A (adopt when m >= n, no copy; a view is copied). */ + SVD(Context& ctx, DeviceMatrix<Scalar>&& d_A, unsigned int options = ComputeThinU | ComputeThinV) : solver_ctx_(ctx) { + compute(std::move(d_A), options); + } + ~SVD() = default; SVD(const SVD&) = delete; @@ -148,8 +153,9 @@ /** Decompose a device matrix (move). For m >= n the buffer is adopted and * consumed in place by gesvd — no copy; for m < n a transposed copy is - * unavoidable. */ + * unavoidable. A view is copied: its storage belongs to another object. */ SVD& compute(DeviceMatrix<Scalar>&& d_A, unsigned int options = ComputeThinU | ComputeThinV) { + if (d_A.isView()) return compute(static_cast<const DeviceMatrix<Scalar>&>(d_A), options); if (!begin_compute(d_A, options)) return *this; if (transposed_) { @@ -173,8 +179,7 @@ eigen_assert(solver_ctx_.info() == Success); const Index k = (std::min)(m_, n_); RealVector S(k); - EIGEN_CUDA_RUNTIME_CHECK( - cudaMemcpy(S.data(), d_S_.get(), static_cast<size_t>(k) * sizeof(RealScalar), cudaMemcpyDeviceToHost)); + solver_ctx_.download(S.data(), d_S_.get(), static_cast<size_t>(k) * sizeof(RealScalar)); return S; } @@ -188,16 +193,13 @@ if (!transposed_) { const Index ucols = (options_ & ComputeFullU) ? m_ : k; PlainMatrix U(m_, ucols); - EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy(U.data(), d_U_.get(), - static_cast<size_t>(m_) * static_cast<size_t>(ucols) * sizeof(Scalar), - cudaMemcpyDeviceToHost)); + solver_ctx_.download(U.data(), d_U_.get(), static_cast<size_t>(m_) * static_cast<size_t>(ucols) * sizeof(Scalar)); return U; } else { const Index vtrows = (options_ & ComputeFullU) ? m_orig : k; PlainMatrix VT_stored(vtrows, n_); - EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy(VT_stored.data(), d_VT_.get(), - static_cast<size_t>(vtrows) * static_cast<size_t>(n_) * sizeof(Scalar), - cudaMemcpyDeviceToHost)); + solver_ctx_.download(VT_stored.data(), d_VT_.get(), + static_cast<size_t>(vtrows) * static_cast<size_t>(n_) * sizeof(Scalar)); return VT_stored.adjoint(); } } @@ -215,16 +217,14 @@ if (!transposed_) { const Index vtrows = (options_ & ComputeFullV) ? n_ : k; PlainMatrix VT(vtrows, n_); - EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy(VT.data(), d_VT_.get(), - static_cast<size_t>(vtrows) * static_cast<size_t>(n_) * sizeof(Scalar), - cudaMemcpyDeviceToHost)); + solver_ctx_.download(VT.data(), d_VT_.get(), + static_cast<size_t>(vtrows) * static_cast<size_t>(n_) * sizeof(Scalar)); return VT; } else { const Index ucols = (options_ & ComputeFullV) ? n_orig : k; PlainMatrix U_stored(m_, ucols); - EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy(U_stored.data(), d_U_.get(), - static_cast<size_t>(m_) * static_cast<size_t>(ucols) * sizeof(Scalar), - cudaMemcpyDeviceToHost)); + solver_ctx_.download(U_stored.data(), d_U_.get(), + static_cast<size_t>(m_) * static_cast<size_t>(ucols) * sizeof(Scalar)); return U_stored.adjoint(); } }
diff --git a/contrib/Eigen/src/GPU/GpuSolverContext.h b/contrib/Eigen/src/GPU/GpuSolverContext.h index 33b05cf..8bfa9ee 100644 --- a/contrib/Eigen/src/GPU/GpuSolverContext.h +++ b/contrib/Eigen/src/GPU/GpuSolverContext.h
@@ -196,6 +196,14 @@ sync_info(); return info_; } + + // Blocking download of solver-owned device data. It waits only for the solver's stream; cudaMemcpy would run on + // the legacy default stream and wait for every blocking stream on the device. + void download(void* dst, const void* src, size_t bytes) const { + if (bytes == 0) return; + EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(dst, src, bytes, cudaMemcpyDeviceToHost, stream())); + EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream())); + } }; } // namespace internal
diff --git a/contrib/Eigen/src/GPU/GpuSparseContext.h b/contrib/Eigen/src/GPU/GpuSparseContext.h index 3de9c32..3fbb1c9 100644 --- a/contrib/Eigen/src/GPU/GpuSparseContext.h +++ b/contrib/Eigen/src/GPU/GpuSparseContext.h
@@ -140,7 +140,8 @@ /** Sparse product expression: DeviceSparseView * DeviceMatrix → SpMVExpr. * Evaluated by DeviceMatrix::operator=(SpMVExpr): dispatches to cusparseSpMV - * when the dense operand has one column, cusparseSpMM otherwise. */ + * when the dense operand has one column, cusparseSpMM otherwise. Adding or + * subtracting a DeviceMatrix yields an SpMVAffineExpr. */ template <typename Scalar_> class SpMVExpr { public: @@ -154,10 +155,73 @@ const DeviceMatrix<Scalar>& x_; }; +/** alpha * (DeviceSparseView * DeviceMatrix) + beta * addend with alpha, beta = ±1: the + * result of `d_b - d_A * d_x`, `d_b + d_A * d_x`, `d_A * d_x + d_b` and `d_A * d_x - d_b`, + * so that Eigen's iterative solver templates (`residual = rhs - mat * x`) compile. + * Evaluated by DeviceMatrix::operator=(SpMVAffineExpr): the addend is copied into the + * destination unless it already is the destination, then one cusparseSpMV (one column) + * or cusparseSpMM call runs with the given alpha and beta. No operator takes an + * SpMVAffineExpr, so an expression with a second addend does not compile. */ +template <typename Scalar_> +class SpMVAffineExpr { + public: + using Scalar = Scalar_; + SpMVAffineExpr(const SpMVExpr<Scalar>& product, Scalar alpha, Scalar beta, const DeviceMatrix<Scalar>& addend) + : view_(product.view()), x_(product.x()), alpha_(alpha), beta_(beta), addend_(addend) {} + const DeviceSparseView<Scalar>& view() const { return view_; } + const DeviceMatrix<Scalar>& x() const { return x_; } + Scalar alpha() const { return alpha_; } + Scalar beta() const { return beta_; } + const DeviceMatrix<Scalar>& addend() const { return addend_; } + + private: + const DeviceSparseView<Scalar>& view_; + const DeviceMatrix<Scalar>& x_; + Scalar alpha_; + Scalar beta_; + const DeviceMatrix<Scalar>& addend_; +}; + +template <typename S> +SpMVAffineExpr<S> operator-(const DeviceMatrix<S>& b, const SpMVExpr<S>& p) { + return SpMVAffineExpr<S>(p, S(-1), S(1), b); +} + +template <typename S> +SpMVAffineExpr<S> operator+(const DeviceMatrix<S>& b, const SpMVExpr<S>& p) { + return SpMVAffineExpr<S>(p, S(1), S(1), b); +} + +template <typename S> +SpMVAffineExpr<S> operator+(const SpMVExpr<S>& p, const DeviceMatrix<S>& b) { + return b + p; +} + +template <typename S> +SpMVAffineExpr<S> operator-(const SpMVExpr<S>& p, const DeviceMatrix<S>& b) { + return SpMVAffineExpr<S>(p, S(1), S(-1), b); +} + +} // namespace gpu + +namespace internal { +// DeviceSparseView is a matrix-free matrix type for Eigen's iterative solvers: it looks like a +// SparseMatrix to the traits machinery, and IterativeSolverBase stores it by pointer because +// Ref<> cannot bind to it (the same mechanism as the matrix-free example in the documentation). +template <typename Scalar_> +struct traits<gpu::DeviceSparseView<Scalar_>> : traits<SparseMatrix<Scalar_, ColMajor, int>> {}; +} // namespace internal + +namespace gpu { + /** Device-resident sparse matrix view. Returned by SparseContext::deviceView() * for a SparseMatrix (CSC) or a BlockSparseMatrix (BSR). Lightweight handle * referencing the context's cached device data. * + * It is also a matrix-free matrix type for Eigen's iterative solvers: + * `ConjugateGradient<DeviceSparseView<Scalar>, Lower | Upper>` runs Eigen's own + * algorithm on device vectors through `solveWithGuessInPlace()`; see the README. + * * \warning One SparseContext caches one sparse matrix at a time. Any later * upload through the same context — a second deviceView() or any host-input * multiply — replaces the cached data and invalidates earlier views; a stale @@ -167,10 +231,16 @@ * * Supports `d_y = d_A * d_x` (SpMV) and `d_Y = d_A * d_X` (SpMM). */ template <typename Scalar_> -class DeviceSparseView { +class DeviceSparseView : public EigenBase<DeviceSparseView<Scalar_>> { public: using Scalar = Scalar_; + using RealScalar = typename NumTraits<Scalar>::Real; + using StorageIndex = int; using SpMat = SparseMatrix<Scalar, ColMajor, int>; + // What IterativeSolverBase and ConjugateGradient read from a matrix type. + static constexpr int ColsAtCompileTime = Dynamic; + static constexpr int MaxColsAtCompileTime = Dynamic; + static constexpr bool IsRowMajor = false; DeviceSparseView(SparseContext<Scalar>& ctx, Index rows, Index cols, uint64_t generation) : ctx_(ctx), rows_(rows), cols_(cols), generation_(generation) {} @@ -180,6 +250,8 @@ Index rows() const { return rows_; } Index cols() const { return cols_; } + /** Number of stored entries of the cached matrix. */ + Index nonZeros() const { return ctx_.nonZeros(); } const SparseContext<Scalar>& context() const { return ctx_; } /** Upload generation this view was created against. Used to detect stale @@ -255,6 +327,9 @@ * every sparse upload (deviceView() or a host-input multiply). */ uint64_t uploadGeneration() const { return generation_; } + /** Number of stored entries of the cached sparse matrix, -1 before the first upload. */ + Index nonZeros() const { return cached_nnz_; } + /** Compute y = A * x. Returns y as a new dense vector. */ template <typename InputType, typename Rhs> DenseVector multiply(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& x) { @@ -938,14 +1013,14 @@ } // Destroy the sparse-matrix descriptor and reset the cache identity. - // `checked` selects assert-on-failure (mid-lifetime rebuilds) vs swallow + // `checked` selects checking (mid-lifetime rebuilds) vs swallowing failure // (noexcept destructor). void destroy_spmat_descriptor(bool checked) { if (spmat_desc_) { - cusparseStatus_t s = cusparseDestroySpMat(spmat_desc_); - eigen_assert((!checked || s == CUSPARSE_STATUS_SUCCESS) && "cusparseDestroySpMat failed"); - EIGEN_UNUSED_VARIABLE(s); - EIGEN_UNUSED_VARIABLE(checked); + if (checked) + EIGEN_CUSPARSE_CHECK(cusparseDestroySpMat(spmat_desc_)); + else + (void)cusparseDestroySpMat(spmat_desc_); spmat_desc_ = nullptr; } set_descriptor_key(-1, -1, -1, 0); @@ -978,9 +1053,40 @@ } template <typename Scalar_> +DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const SpMVAffineExpr<Scalar_>& expr) { + const DeviceSparseView<Scalar_>& view = expr.view(); + const DeviceMatrix& addend = expr.addend(); + eigen_assert(view.generation() == view.context().uploadGeneration() && + "DeviceSparseView is stale: its SparseContext has since uploaded another sparse matrix"); + eigen_assert(addend.rows() == view.rows() && addend.cols() == expr.x().cols() && + "SpMVAffineExpr: the addend must have the shape of the product"); + eigen_assert(&expr.x() != this && "SpMVAffineExpr: the destination aliases the dense operand"); + // The product accumulates into a copy of the addend; when the addend is the destination + // itself (d_r = d_r - d_A * d_x) cuSPARSE's beta accumulates in place. + if (&addend != this) copyFrom(Context::threadLocal(), addend); + // With no stored entries the expression is beta * addend, which spmv_device_exec cannot + // form: SparseContext owns no cuBLAS handle to scale d_y by beta. + if (view.nonZeros() == 0 || view.rows() == 0 || view.cols() == 0) { + if (expr.beta() != Scalar_(1)) scale(Context::threadLocal(), expr.beta()); + return *this; + } + if (expr.x().cols() <= 1) { + view.context().spmv_device_exec(expr.x(), *this, expr.alpha(), expr.beta(), GpuOp::NoTrans); + } else { + view.context().spmm_device_exec(expr.x(), *this, expr.alpha(), expr.beta(), GpuOp::NoTrans); + } + return *this; +} + +template <typename Scalar_> DeviceMatrix<Scalar_>::DeviceMatrix(const SpMVExpr<Scalar_>& expr) : DeviceMatrix() { *this = expr; } + +template <typename Scalar_> +DeviceMatrix<Scalar_>::DeviceMatrix(const SpMVAffineExpr<Scalar_>& expr) : DeviceMatrix() { + *this = expr; +} } // namespace gpu } // namespace Eigen
diff --git a/contrib/Eigen/src/GPU/GpuSupport.h b/contrib/Eigen/src/GPU/GpuSupport.h index ef0ee6b..4414f36 100644 --- a/contrib/Eigen/src/GPU/GpuSupport.h +++ b/contrib/Eigen/src/GPU/GpuSupport.h
@@ -18,6 +18,8 @@ #include "./InternalHeaderCheck.h" #include <cuda_runtime.h> +#include <cstdio> +#include <cstdlib> #include <vector> #include <limits> @@ -32,15 +34,43 @@ enum class GpuOp { NoTrans, Trans, ConjTrans }; namespace internal { -// Aborts via eigen_assert on failure, and eigen_assert throws where it is so -// configured, so this must not be used in a destructor or any other noexcept -// function: there the throw would call std::terminate. -#define EIGEN_CUDA_RUNTIME_CHECK(expr) \ - do { \ - cudaError_t _e = (expr); \ - eigen_assert(_e == cudaSuccess && "CUDA runtime call failed"); \ +// Prints `file:line: call: error` to stderr and stops: std::abort() where +// assertions are compiled out, a failed eigen_assert otherwise. No build goes +// on past a failed call: its work was not done, and a sticky error leaves the +// context unusable for every later call. +inline void gpu_check_failed(const char* error, const char* expression, const char* file, int line) { + std::fprintf(stderr, "%s:%d: %s: %s\n", file, line, expression, error); +#if defined(EIGEN_NO_DEBUG) + std::abort(); +#else + eigen_assert(false && "GPU runtime or library call failed"); +#endif +} + +// Every check macro of the module reports failures here, in every build. It may +// be defined to throw, so no destructor or noexcept function uses the checks. +#ifndef EIGEN_GPU_CHECK_FAILED +#define EIGEN_GPU_CHECK_FAILED(error, expression, file, line) \ + ::Eigen::gpu::internal::gpu_check_failed(error, expression, file, line) +#endif + +#define EIGEN_CUDA_RUNTIME_CHECK(expr) \ + do { \ + const cudaError_t _e = (expr); \ + if (_e != cudaSuccess) EIGEN_GPU_CHECK_FAILED(cudaGetErrorName(_e), #expr, __FILE__, __LINE__); \ } while (0) +// For the libraries without a status-to-string function: "<library> status <code>". +inline void gpu_check_failed_code(const char* library, int status, const char* expression, const char* file, int line) { + char error[64]; + std::snprintf(error, sizeof(error), "%s status %d", library, status); + // A user-defined EIGEN_GPU_CHECK_FAILED need not use every argument. + EIGEN_UNUSED_VARIABLE(expression); + EIGEN_UNUSED_VARIABLE(file); + EIGEN_UNUSED_VARIABLE(line); + EIGEN_GPU_CHECK_FAILED(error, expression, file, line); +} + // cuBLAS and the legacy cuSOLVER APIs take dimensions and leading dimensions as // 32-bit `int`, while Eigen's Index is 64-bit by default and GPU allocations can // exceed INT_MAX in one dimension. Narrow through this helper at every such call
diff --git a/contrib/Eigen/src/GPU/README.md b/contrib/Eigen/src/GPU/README.md index 45e91b3..07e2fdf 100644 --- a/contrib/Eigen/src/GPU/README.md +++ b/contrib/Eigen/src/GPU/README.md
@@ -88,14 +88,17 @@ coefficient-level evaluation, lazy fusion, or packet operations. **Interoperability where useful.** `DeviceMatrix` provides the same operator -signatures as `Matrix` for common vector operations: `+=`, `-=`, `*=`, -`dot()`, `squaredNorm()`, `norm()`, `setZero()`, and `noalias()`. This makes +signatures as `Matrix` for common vector operations: `+=`, `-=`, `*=`, `/=`, +`dot()`, `squaredNorm()`, `norm()`, `stableNorm()`, `setZero()`, `noalias()`, +and copy construction and assignment (a device-to-device copy). This makes `DeviceMatrix` usable as a drop-in `VectorType` in Eigen algorithm templates -that rely on these operations. For example, Eigen's `conjugate_gradient()` -template works with `DeviceMatrix` with a single typedef change -- no -modifications to the algorithm or the expression template system. Conjugate -gradient is just the motivating example; we are open to expanding operator -coverage as needed to support other high-level Eigen algorithms on the GPU. +that rely on these operations, and `DeviceSparseView` is a matrix-free matrix +type for Eigen's iterative solvers: `ConjugateGradient<gpu::DeviceSparseView<double>, +Lower | Upper>` runs Eigen's own algorithm, unmodified, on device vectors (see +[Eigen algorithm interop](#eigen_gpu_interop)). +Conjugate gradient is just the motivating example; we are open to expanding +operator coverage as needed to support other high-level Eigen algorithms on the +GPU. **Explicit over implicit.** Host-device transfers, stream management, and library handle lifetimes are visible in the API. There are no hidden @@ -150,6 +153,11 @@ d_x += alpha * d_p; // host scalar * DeviceMatrix (axpy) ``` +Every reduction also has an overload that writes into an existing +`DeviceScalar` on a given Context, `d_x.dot(ctx, d_y, s)`, where `s` lives on +`ctx.stream()`: it reuses the scalar's storage, so a loop, or a captured CUDA +graph, repeats the reduction without allocating. + Division between `DeviceScalar` values (real types only) is performed on device via NPP, avoiding extra synchronizations. Small device allocations (including `DeviceScalar`) go through the stream-ordered allocator like every @@ -218,6 +226,7 @@ | Dense solvers (LLT, LU, QR, SVD, EVD) | `-lcusolver -lcublas` | | FFT (`gpu::FFT`) | `-lcufft -lcublas` | | SpMV / SpMM (`gpu::SparseContext`) | `-lcusparse -lcublas` | +| `norm()`, `DeviceScalar` arithmetic, `/=`, `cwiseProduct` | `-lnpps -lnppc` | | Sparse direct solvers (cuDSS) | `-lcudss -lcublas` | cuBLAS is required by `DeviceMatrix` itself (every `Context` creates a cuBLAS @@ -267,14 +276,22 @@ ```cpp // Dot product and norms (return DeviceScalar -- no sync until read) auto dot_val = d_x.dot(d_y); // cublasDdot / cublasCdotc -auto norm_val = d_r.norm(); // cublasDnrm2 +auto norm_val = d_r.norm(); // sqrt(dot): cublasDdot, then NPP sqrt on device double n = norm_val; // implicit conversion triggers sync // Vector arithmetic (cuBLAS axpy / geam) d_x += alpha * d_p; // axpy: x = x + alpha * p d_x -= alpha * d_p; // axpy: x = x - alpha * p d_x *= alpha; // scal: x = alpha * x +d_x /= alpha; // NPP divide-by-constant: x = x / alpha (true division for real Scalar) d_r.setZero(); // cudaMemsetAsync +auto s = d_r.stableNorm(); // cublasDnrm2: scaled, overflow-safe + +// Copies are device-to-device (cuBLAS copy) on the thread-local Context; no host round trip. +// They exist so that existing Eigen algorithm code runs on the GPU unchanged; code written for +// the GPU should avoid them (move, or reuse a buffer with copyFrom), since each is a transfer. +gpu::DeviceMatrix<double> d_q = d_p; // copy construction +d_q = d_r; // copy assignment, resizes if needed // DeviceScalar arithmetic (stays on device, real types only) auto alpha = absNew / dot_val; // device-side division via NPP @@ -316,6 +333,11 @@ expression fully asynchronous — use the cached `gpu::LLT` / `gpu::LU` classes and `info()` when numerical failure must be detected. +cuSOLVER 11.4.1 and 11.4.2 (CUDA 11.8 and 12.0) have a defect here: +`cusolverDnXpotrf` reports success on a matrix that is not positive definite +and returns a factor full of NaN, so `gpu::LLT::info()` cannot detect that +failure on those versions. cuSOLVER 11.4.4 (CUDA 12.1) and later report it. + **Cached factorization** -- Factor once, solve many times: ```cpp @@ -370,7 +392,8 @@ The cached API keeps the factored matrix on device, avoiding redundant host-device transfers and re-factorizations. All five solvers accept `compute(DeviceMatrix&&)` to adopt the input and factor it in place with no -copy (for QR/SVD with m < n the internal transpose still copies), and all five +copy (for QR/SVD with m < n the internal transpose still copies, and a view is +always copied because its storage belongs to another object), and all five can bind to a `gpu::Context` to share its stream and handles. All solvers also accept host dense expressions directly as a convenience (e.g., `gpu::LLT<double> llt(A)` or `qr.solve(B)`), which handles upload/download @@ -528,6 +551,8 @@ auto d_A = spmv_dev.deviceView(A); // upload sparse matrix once d_y = d_A * d_x; // SpMV, stays on device d_Y = d_A * d_X; // SpMM when the RHS has > 1 column +d_r = d_b - d_A * d_x; // residual: copy of d_b, then one SpMV with beta = 1 +d_r = d_r - d_A * d_x; // in place when the addend is the destination: no copy ``` Host-input calls re-upload the sparse values *and* index arrays on every call @@ -538,7 +563,7 @@ carries a generation counter — using a view after any later upload through its context asserts instead of silently multiplying by the wrong matrix. -#### Block sparse matrices (BSR) +#### Block sparse matrices (BSR) {#eigen_gpu_bsr} Every `SparseContext` entry point above also accepts a `BlockSparseMatrix`. Square blocks of size at least 2 upload in cuSPARSE's BSR (block sparse row) @@ -577,14 +602,51 @@ `int` index type and the `int` limits on dimensions and nonzeros are as for `SparseMatrix`. -### Eigen algorithm interop (example: Conjugate gradient) +### Eigen algorithm interop (example: Conjugate gradient) {#eigen_gpu_interop} -The BLAS-1 operators and `DeviceSparseView` make `DeviceMatrix` usable as a -vector type in GPU implementations of algorithms like conjugate gradient. -Conjugate gradient is the motivating example -- the GPU CG mirrors Eigen's -`conjugate_gradient()` line for line, with only one host sync per iteration -(the convergence check). All scalar intermediates (`alpha`, `beta`, `absNew`) -stay on device as `DeviceScalar` values: +Eigen's `ConjugateGradient` runs on the GPU types. `DeviceSparseView` is a +matrix-free matrix type (it inherits `EigenBase` and carries `SparseMatrix` +traits, so `IterativeSolverBase` holds it by pointer), the vectors are +`DeviceMatrix`, and the algorithm's `VectorType` (`Dest::PlainObject`) is +`DeviceMatrix` itself. `solve()` and `solveWithGuess()` return Eigen +expressions and need Eigen operands; `solveWithGuessInPlace(b, x)` is the entry +point for device vectors, with `x` holding the initial guess. The identity +preconditioner works as is; `DiagonalPreconditioner` and the other Eigen +preconditioners evaluate host expressions and do not. `compute()` stores a +pointer to the view, so the view and the `SparseContext` behind it must outlive +the solver. The class form is real-`Scalar` only: `DeviceScalar` arithmetic +covers real types, and `numext::real()` of a complex `DeviceScalar` has no host +conversion to `RealScalar`. + +```cpp +gpu::Context ctx; +gpu::Context::setThreadLocal(&ctx); +gpu::SparseContext<double> spmv(ctx); +auto mat = spmv.deviceView(A); +auto d_b = gpu::DeviceMatrix<double>::fromHost(b, ctx.stream()); +gpu::DeviceMatrix<double> d_x(n, 1); +d_x.setZero(ctx); + +ConjugateGradient<gpu::DeviceSparseView<double>, Lower | Upper, IdentityPreconditioner> cg; +cg.setTolerance(1e-10); +cg.compute(mat); // matrix-free: stores a pointer to mat +cg.solveWithGuessInPlace(d_b, d_x); // Eigen's algorithm, cuSPARSE and cuBLAS underneath +// cg.info(), cg.iterations(), cg.error() as usual +gpu::Context::setThreadLocal(nullptr); +``` + +The algorithm reads three values on the host per iteration -- `alpha`, the +`stableNorm()` convergence check and `absNew` -- so this form synchronizes three +times per iteration. The division `absNew / p.dot(tmp)` resolves to the +device-side `operator/(Scalar, DeviceScalar)`: `absNew` is uploaded into a +fresh `DeviceScalar`, divided through NPP and read back, two small allocations +and a kernel launch per iteration on top of the sync. The hand-written loop +below is the same algorithm with one host sync per iteration (the convergence +check); all scalar intermediates (`alpha`, `beta`, `absNew`) stay on device as +`DeviceScalar` values. Its convergence test squares the residual norm +(`squaredNorm()`, a cuBLAS dot), which overflows for `||r|| > sqrt(max)` and +underflows to 0 for `||r|| < sqrt(min)`, so it lacks the extreme-scale +robustness the template gets from `stableNorm()` and its residual scaling: ```cpp gpu::Context ctx; @@ -690,6 +752,44 @@ - `DeviceMatrix::resize()` is capacity-aware: shrinking or same-size reshapes reuse the existing allocation (contents are still discarded). +### Error handling {#eigen_gpu_errors} + +Every CUDA runtime and library call the module makes is checked, in release +builds as in debug builds. A failed call prints `file:line: call: error` to +`stderr` and stops the program: + +- `std::abort()` when `EIGEN_NO_DEBUG` (or `NDEBUG`) is defined; +- a failed `eigen_assert` otherwise. + +`error` is the status name where the library provides one +(`cudaErrorInvalidValue`, `CUBLAS_STATUS_INVALID_VALUE`, ...) and +`<library> status <code>` for cuFFT, cuDSS, NPP and cuBLAS before 11.6.1. +There is no mode that ignores a failure: the failed call has not done its work, +and a sticky error (say, an illegal address in a kernel) makes every later call +on the device fail as well. + +To handle failures yourself, define +`EIGEN_GPU_CHECK_FAILED(error, expression, file, line)` before including the +module. `error`, `expression` and `file` are C strings; `line` is an `int`. For +example, to turn failures into exceptions: + +```cpp +#define EIGEN_GPU_CHECK_FAILED(error, expression, file, line) \ + throw std::runtime_error(std::string(file) + ": " + (expression) + ": " + (error)) +#include <contrib/Eigen/GPU> +``` + +Destructors release their resources without the checks, so a throwing handler +never runs inside one. A throw also restores the library-handle state an +operation changes temporarily (the cuBLAS pointer mode), so the context stays +usable; only the interrupted operation's output is unspecified. A handler that +returns lets execution continue past the failed call, which is only useful in +tests. + +Numerical failures (a matrix that is not positive definite, a singular +factorization) are not call failures: they are reported by `info()` as +described above. + ## Reference ### Supported scalar types @@ -727,8 +827,9 @@ | `x -= alpha * y` | `cublasXaxpy` | alpha negated | | `x *= alpha` | `cublasXscal` | alpha (host or DeviceScalar) | | `x.dot(y)` | `cublasXdot` / `cublasXdotc` | returns `DeviceScalar` | -| `x.norm()` | `cublasXnrm2` | returns `DeviceScalar<RealScalar>` | -| `x.squaredNorm()` | `cublasXdot(x, x)` | returns `DeviceScalar<RealScalar>` | +| `x.norm()` | `cublasXdot(x, x)`, then `nppsSqrt` | as `squaredNorm()`, then its square root on device | +| `x.stableNorm()` | `cublasXnrm2` | returns `DeviceScalar<RealScalar>` | +| `x.squaredNorm()` | `cublasXdot(x, x)` | real dot over the `2n` real and imaginary parts for complex `x`; returns `DeviceScalar<RealScalar>` | | `d_y = view * d_x` | `cusparseSpMV` | device-resident SpMV | | `d_Y = view * d_X` | `cusparseSpMM` | device-resident SpMM (RHS with >1 column) | | same, `view` of a `BlockSparseMatrix` | `cusparseSpMV` / `cusparseSpMM` on a BSR descriptor | opA=N, row-major blocks; op(A) formed on the host | @@ -762,6 +863,7 @@ Index cols() size_t sizeInBytes() bool empty() +bool isView() // Holds storage borrowed through view() Scalar* data() // Raw device pointer void resize(Index rows, Index cols) // Discard contents; keeps the allocation // when it is already large enough @@ -778,8 +880,11 @@ // BLAS Level-1 (all have overloads with explicit gpu::Context& parameter) DeviceScalar<Scalar> dot(const DeviceMatrix& other) // cuBLAS dot/dotc -> DeviceScalar -DeviceScalar<RealScalar> norm() // cuBLAS nrm2 -> DeviceScalar +DeviceScalar<RealScalar> norm() // sqrt(squaredNorm()) -> DeviceScalar, unscaled like MatrixBase::norm() +DeviceScalar<RealScalar> stableNorm() // cuBLAS nrm2 (scaled, overflow-safe) -> DeviceScalar DeviceScalar<RealScalar> squaredNorm() // dot(self, self) -> DeviceScalar (no sync) +void dot(ctx, other, DeviceScalar<Scalar>& result) // Each reduction into an existing DeviceScalar +void squaredNorm / norm / stableNorm(ctx, result) // on ctx's stream, reusing its storage: no allocation void setZero() // cudaMemsetAsync void addScaled(gpu::Context&, Scalar alpha, const DeviceMatrix& x) // this += alpha * x (axpy) void scale(gpu::Context&, Scalar alpha) // this *= alpha (scal) @@ -839,6 +944,7 @@ cusolverDnHandle_t cusolverHandle() // Lazy: creates the handle on first call cublasLtHandle_t cublasLtHandle() // Lazy-initialized cusparseHandle_t cusparseHandle() // Lazy-initialized +const NppStreamContext& nppStreamContext() // NPP context for stream(), filled in at construction internal::DeviceBuffer& gemmWorkspace() // cublasLtMatmul scratch (lazy-grown per context) internal::CublasLtPlanCache& gemmPlanCache() // shape-keyed plan cache (per context, ~8-entry LRU) @@ -863,11 +969,11 @@ gpu::LLT& compute(const DenseBase<D>& A) // Upload + factorize gpu::LLT& compute(const DeviceMatrix& d_A) // D2D copy + factorize -gpu::LLT& compute(DeviceMatrix&& d_A) // Adopt + factorize (no copy) +gpu::LLT& compute(DeviceMatrix&& d_A) // Adopt + factorize (a view is copied) PlainMatrix solve(const MatrixBase<D>& B) // -> host Matrix (syncs) DeviceMatrix solve(const DeviceMatrix& d_B) // -> DeviceMatrix (async, stays on device) -DeviceMatrix solve(DeviceMatrix&& d_B) // In-place: consumes RHS, no copy/alloc +DeviceMatrix solve(DeviceMatrix&& d_B) // In-place: consumes RHS, no copy/alloc (a view is copied) ComputationInfo info() // Lazy sync on first call: Success or NumericalIssue Index rows() / cols() @@ -892,10 +998,15 @@ ```cpp gpu::QR() // Default construct +gpu::QR(Context& ctx) // Bind to ctx's stream + handles gpu::QR(const DenseBase<D>& A) // Convenience: upload + factorize +gpu::QR(const DeviceMatrix& d_A) // Convenience: D2D copy + factorize +gpu::QR(DeviceMatrix&& d_A) // Convenience: adopt (m >= n) + factorize +gpu::QR(Context& ctx, ...) // Bind + factorize in one step gpu::QR& compute(const DenseBase<D>& A) // Upload + factorize gpu::QR& compute(const DeviceMatrix& d_A) // D2D copy + factorize +gpu::QR& compute(DeviceMatrix&& d_A) // Adopt + factorize (copies when m < n or for a view) PlainMatrix solve(const MatrixBase<D>& B) // -> host Matrix (syncs) DeviceMatrix solve(const DeviceMatrix& d_B) // -> DeviceMatrix (async) @@ -914,10 +1025,15 @@ ```cpp gpu::SVD() // Default construct, then call compute() +gpu::SVD(Context& ctx) // Bind to ctx's stream + handles gpu::SVD(const DenseBase<D>& A, unsigned options = ComputeThinU | ComputeThinV) // Convenience +gpu::SVD(const DeviceMatrix& d_A, unsigned options = ComputeThinU | ComputeThinV) // D2D copy +gpu::SVD(DeviceMatrix&& d_A, unsigned options = ComputeThinU | ComputeThinV) // Adopt (m >= n) +gpu::SVD(Context& ctx, ...) // Bind + decompose in one step gpu::SVD& compute(const DenseBase<D>& A, unsigned options = ComputeThinU | ComputeThinV) gpu::SVD& compute(const DeviceMatrix& d_A, unsigned options = ComputeThinU | ComputeThinV) +gpu::SVD& compute(DeviceMatrix&& d_A, unsigned options = ComputeThinU | ComputeThinV) RealVector singularValues() // -> host vector (syncs, downloads) PlainMatrix matrixU() // -> host Matrix (syncs, downloads) @@ -950,14 +1066,19 @@ ### `gpu::SelfAdjointEigenSolver<Scalar>` -- Eigendecomposition (cuSOLVER) Symmetric/Hermitian eigenvalue decomposition via `cusolverDnXsyevd`. -`ComputeMode` enum: `EigenvaluesOnly`, `ComputeEigenvectors`. +`options`: `ComputeEigenvectors` (the default) or `EigenvaluesOnly`. ```cpp gpu::SelfAdjointEigenSolver() // Default construct, then call compute() -gpu::SelfAdjointEigenSolver(const DenseBase<D>& A, ComputeMode mode = ComputeEigenvectors) // Convenience +gpu::SelfAdjointEigenSolver(Context& ctx) // Bind to ctx's stream + handles +gpu::SelfAdjointEigenSolver(const DenseBase<D>& A, int options = ComputeEigenvectors) // Convenience +gpu::SelfAdjointEigenSolver(const DeviceMatrix& d_A, int options = ComputeEigenvectors) // D2D copy +gpu::SelfAdjointEigenSolver(DeviceMatrix&& d_A, int options = ComputeEigenvectors) // Adopt (a view is copied) +gpu::SelfAdjointEigenSolver(Context& ctx, ...) // Bind + decompose in one step -gpu::SelfAdjointEigenSolver& compute(const DenseBase<D>& A, ComputeMode mode = ComputeEigenvectors) -gpu::SelfAdjointEigenSolver& compute(const DeviceMatrix& d_A, ComputeMode mode = ComputeEigenvectors) +gpu::SelfAdjointEigenSolver& compute(const DenseBase<D>& A, int options = ComputeEigenvectors) +gpu::SelfAdjointEigenSolver& compute(const DeviceMatrix& d_A, int options = ComputeEigenvectors) +gpu::SelfAdjointEigenSolver& compute(DeviceMatrix&& d_A, int options = ComputeEigenvectors) RealVector eigenvalues() // -> host vector (syncs, downloads, ascending order) PlainMatrix eigenvectors() // -> host Matrix (syncs, downloads, columns) @@ -1072,7 +1193,7 @@ Accepts `SparseMatrix<Scalar, ColMajor>` and `BlockSparseMatrix<Scalar, Options, BlockRows, BlockCols, int>` (the `BlockSpMat<Options, BlockRows, -BlockCols>` alias; see [Block sparse matrices](#block-sparse-matrices-bsr) for +BlockCols>` alias; see [Block sparse matrices](#eigen_gpu_bsr) for which block shapes upload as BSR and which op / storage-order combinations do so without a host copy). Host-input methods accept host data and return host data; device-input methods @@ -1163,15 +1284,15 @@ | File | Depends on | Contents | |------|-----------|----------| -| `GpuSupport.h` | `<cuda_runtime.h>` | Error macro, `DeviceBuffer`, `DeviceBufferPool`, `cuda_data_type<>` | +| `GpuSupport.h` | `<cuda_runtime.h>` | `EIGEN_GPU_CHECK_FAILED`, runtime error macro, `DeviceBuffer`, `DeviceBufferPool`, `cuda_data_type<>` | | `DeviceMatrix.h` | `GpuSupport.h` | `gpu::DeviceMatrix<>`, `gpu::HostTransfer<>` | | `DeviceExpr.h` | `DeviceMatrix.h` | GEMM, geam, and device-scalar expression wrappers | | `DeviceBlasExpr.h` | `DeviceMatrix.h` | TRSM, SYMM, SYRK expression wrappers | | `DeviceSolverExpr.h` | `DeviceMatrix.h` | Solver expression wrappers (LLT, LU) | | `DeviceScalar.h` | `GpuSupport.h`, `DeviceScalarOps.h` | `gpu::DeviceScalar<>` (device-resident scalar) | -| `DeviceScalarOps.h` | `<npps_*.h>` | Scalar div/neg/cwiseProduct via NPP | +| `DeviceScalarOps.h` | `<npps_*.h>` | Scalar div/neg/sqrt/cwiseProduct via NPP, NPP error macro | | `DeviceDispatch.h` | all above | All dispatch functions, BLAS-1 out-of-line defs, `gpu::Assignment` | -| `GpuContext.h` | `CuBlasSupport.h`, `CuSolverSupport.h` | `gpu::Context` | +| `GpuContext.h` | `CuBlasSupport.h`, `CuSolverSupport.h`, `CuSparseSupport.h` | `gpu::Context` | | `CuBlasSupport.h` | `GpuSupport.h`, `<cublas_v2.h>`, `<cublasLt.h>` | cuBLAS error macro, type-specific wrappers | | `CuSolverSupport.h` | `GpuSupport.h`, `<cusolverDn.h>` | cuSOLVER params, fill-mode mapping | | `GpuSolverContext.h` | `CuSolverSupport.h`, `CuBlasSupport.h` | Shared solver context (stream, handles, scratch) | @@ -1195,9 +1316,7 @@ ```bash cmake -G Ninja -B build -S . \ -DEIGEN_TEST_CUDA=ON \ - -DEIGEN_CUDA_COMPUTE_ARCH="70" \ - -DEIGEN_TEST_CUBLAS=ON \ - -DEIGEN_TEST_CUSOLVER=ON + -DEIGEN_CUDA_COMPUTE_ARCH="70" cmake --build build --target cublas cusolver_llt cusolver_lu \ cusolver_qr cusolver_svd cusolver_eigen \
diff --git a/contrib/Eigen/src/NumericalDiff/Stencil.h b/contrib/Eigen/src/NumericalDiff/Stencil.h new file mode 100644 index 0000000..4ad646e --- /dev/null +++ b/contrib/Eigen/src/NumericalDiff/Stencil.h
@@ -0,0 +1,166 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#ifndef EIGEN_NUMERICALDIFF_STENCIL_H +#define EIGEN_NUMERICALDIFF_STENCIL_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +/** \class Stencil + * \ingroup NumericalDiff_Module + * + * \brief Finite-difference weights on an arbitrarily spaced grid. + * + * Computes the weights of the order-`Derivative` finite-difference formula for the grid `points`, + * using Fornberg's recursive algorithm: + * + * B. Fornberg, "Generation of Finite Difference Formulas on Arbitrarily Spaced Grids", + * Math. Comp. 51 (1988), 699-706. + * + * The implementation follows the equivalent, in-place algorithmic form given in the follow-up: + * + * B. Fornberg, "Calculation of Weights in Finite Difference Formulas", SIAM Review 40 (1998), 685-691. + * + * The evaluation point is fixed at 0: `points` are read as offsets from wherever the derivative is + * to be evaluated. This is lossless, since shifting every point (and the evaluation point) by the + * same amount leaves the weights unchanged, so a caller evaluating at `x` shifts `points` by `-x` + * first. `Derivative == 0` is supported and corresponds to interpolation: the weights reproduce + * `f(0)` exactly for every polynomial of degree less than `Size`. + * + * `points` are used in exactly the order given: this class does not sort or otherwise reorder them. + * Fornberg's paper notes that the order nodes are introduced in affects the conditioning of the + * recursion (not the exact result, by uniqueness of the interpolating polynomial), and suggests + * introducing nodes in order of ascending distance from the evaluation point for stability. Choosing + * and applying such an ordering, including e.g. a Leja ordering for complex nodes, is the caller's + * responsibility. + * + * For offsets `h * k` at a fixed relative layout `k` and varying step `h`, the weights scale as + * `w_i(h * k) == h^(-Derivative) * w_i(k)`: computing weights once on `k` and scaling by `h^(-Derivative)` + * avoids repeating the `O(Size^2 * Derivative)` recursion for every step. `Stencil` only produces the + * weights for the points it is given; choosing or adapting `h` itself is the caller's responsibility. + * + * \tparam Derivative The order of the derivative the weights approximate. + * \tparam Scalar The scalar type of the grid points and weights. Must be a non-integer type. + * \tparam Size The number of grid points. + */ +template <int Derivative, typename Scalar, int Size> +class Stencil { + EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar) + static_assert(Size >= 0, "Stencil requires a fixed-size point set"); + static_assert(Derivative >= 0 && Derivative < Size, "Stencil requires at least `Derivative + 1` points"); + + public: + static constexpr int DerivativeOrder = Derivative; + static constexpr int PointCount = Size; + + /** The type returned by weights() and points(): a plain, vectorizable array, copied out of the object. */ + using ArrayType = Array<Scalar, Size, 1>; + + /** Computes the weights of the order-\a Derivative finite-difference formula for \a points, given as + * any fixed-size, 1D dense expression of \a Size coefficients convertible to \a Scalar. */ + template <typename Derived> + explicit constexpr Stencil(const DenseBase<Derived>& points) { + static_assert(Derived::IsVectorAtCompileTime, "Stencil requires a 1D dense expression"); + static_assert(Derived::SizeAtCompileTime == Size, "Stencil requires exactly `Size` points"); + for (Index i = 0; i < Size; ++i) m_points[i] = points.coeff(i); + computeWeights(); + } + + /** Constructs from a plain C array of \a Size points. Unlike the DenseBase overload above, this + * constructor does not route through Eigen::Array, so it and weight()/point() remain usable in an + * actual constant expression on compilers where Array's own fixed-size constructors are not. */ + explicit constexpr Stencil(const Scalar (&points)[Size]) { + for (Index i = 0; i < Size; ++i) m_points[i] = points[i]; + computeWeights(); + } + + /** \returns the weights, in the same order as points(). */ + ArrayType weights() const { return Map<const ArrayType>(m_weights); } + + /** \returns the grid points, in the order originally supplied. */ + ArrayType points() const { return Map<const ArrayType>(m_points); } + + /** \returns the weight of points()[i]. Unlike weights()[i], usable in a constant expression: it + * indexes the underlying storage directly rather than through Map, whose constructor is not + * constexpr-evaluable on every supported compiler. */ + constexpr const Scalar& weight(Index i) const { return m_weights[i]; } + + /** \returns points()[i]. Unlike points()[i] (through the Map returned by points()), usable in a + * constant expression, for the same reason as weight(). */ + constexpr const Scalar& point(Index i) const { return m_points[i]; } + + private: + // Fornberg's recursion (1988), in the in-place algorithmic form given by Fornberg (1998). + // `delta[nu][m]` is the weight of `points[nu]` in the order-`m` formula built from `points[0..n]` + // so far; each column `nu` only ever reads/writes its own entries as `n` grows, so a single working + // table can be updated in place. + constexpr void computeWeights() { + Scalar delta[Size][Derivative + 1]{}; + delta[0][0] = Scalar(1); + + for (Index n = 1; n < Size; ++n) { + const Index mn = numext::mini(n, Index(Derivative)); + const Scalar c4 = m_points[n]; + // c1/c2 = product_{j<n-1} ((x_{n-1}-x_j)/(x_n-x_j)) / (x_n-x_{n-1}). + // Form the ratio directly to avoid overflow/underflow of the individual products. + Scalar ratio(1); + for (Index nu = 0; nu < n; ++nu) { + const Scalar c3 = m_points[n] - m_points[nu]; + eigen_assert(!(c3 == Scalar(0)) && "Stencil points must be pairwise distinct"); + if (nu == n - 1) { + ratio /= c3; + delta[n][0] = -ratio * (m_points[n - 1] * delta[n - 1][0]); + for (Index m = mn; m >= 1; --m) + delta[n][m] = ratio * (Scalar(m) * delta[n - 1][m - 1] - m_points[n - 1] * delta[n - 1][m]); + } else { + ratio *= (m_points[n - 1] - m_points[nu]) / c3; + } + for (Index m = mn; m >= 1; --m) delta[nu][m] = (c4 * delta[nu][m] - Scalar(m) * delta[nu][m - 1]) / c3; + delta[nu][0] = c4 * delta[nu][0] / c3; + } + } + + for (Index nu = 0; nu < Size; ++nu) m_weights[nu] = delta[nu][Derivative]; + } + + // NOTE: C arrays instead of `std::array` (through `Eigen::array`) is intentional since + // non-`const` `operator[]` is only `constexpr` in C++17, not C++14. + Scalar m_points[Size]{}; + Scalar m_weights[Size]{}; +}; + +// Redundant out-of-class definitions are required pre-C++17 but deprecated since. +#if EIGEN_COMP_CXXVER < 17 +template <int Derivative, typename Scalar, int Size> +constexpr int Stencil<Derivative, Scalar, Size>::DerivativeOrder; +template <int Derivative, typename Scalar, int Size> +constexpr int Stencil<Derivative, Scalar, Size>::PointCount; +#endif + +/** \relates Stencil + * Deduces \a Scalar and \a Size from \a points; \a Derivative must still be supplied explicitly, + * e.g. `makeStencil<2>(points)`. Exists because C++14 has no class template argument deduction. + */ +template <int Derivative, typename Derived> +constexpr Stencil<Derivative, typename Derived::Scalar, Derived::SizeAtCompileTime> makeStencil( + const DenseBase<Derived>& points) { + return Stencil<Derivative, typename Derived::Scalar, Derived::SizeAtCompileTime>(points); +} + +/** \relates Stencil + * As above, deducing \a Scalar and \a Size from a plain C array of points. + */ +template <int Derivative, typename Scalar, int Size> +constexpr Stencil<Derivative, Scalar, Size> makeStencil(const Scalar (&points)[Size]) { + return Stencil<Derivative, Scalar, Size>(points); +} + +} // namespace Eigen + +#endif
diff --git a/contrib/Eigen/src/StructuredMatrices/Bccb.h b/contrib/Eigen/src/StructuredMatrices/Bccb.h index 0698656..e950f57 100644 --- a/contrib/Eigen/src/StructuredMatrices/Bccb.h +++ b/contrib/Eigen/src/StructuredMatrices/Bccb.h
@@ -26,8 +26,7 @@ // (fraction, exponent) pair to avoid spurious overflow/underflow. // [5] P. H. Sterbenz, "Floating-Point Computation", Prentice-Hall, 1974. // Scaling by a power of two is exact, the property the balanced -// accumulation, the rescaled rank threshold and the scaled 2-D FFT -// products rely on. +// accumulation relies on. #ifndef EIGEN_STRUCTURED_BCCB_H #define EIGEN_STRUCTURED_BCCB_H @@ -171,7 +170,6 @@ m_symbol = computeSymbol(); m_prodSymbol = computeProdSymbol(m_g); } - m_fftUsable = computeFftUsable(); } EIGEN_DEVICE_FUNC Index rows() const { return m_g.size(); } @@ -236,48 +234,23 @@ const Index N = rows(); eigen_assert(b.rows() == N && "right-hand side has the wrong number of rows"); const ComplexArray s = symbol(); - RealArray mods; - RealScalar tol; - scaledModuli(s, mods, tol); - ComplexArray sinv(s.rows(), s.cols()); - // Strictly-below-threshold entries are zeroed, matching SVDBase::rank(), so a - // smallest-normal 1x1 operator stays invertible. A comparison with NaN is - // false, so NaN symbol entries stay in the inverted set and propagate to the - // output instead of being silently zeroed. - // The packet path sanitizes discarded entries before division; exceptional - // packets retain the scalar branch and balanced reciprocal. - internal::structured_symbol_reciprocal(s.data(), mods.data(), tol, sinv.data(), s.size()); + const RealArray mods = s.cwiseAbs(); + const ComplexArray sinv = internal::structured_pinv_symbol(s, mods, internal::structured_rank_threshold(s, mods)); Matrix<Scalar, RowsAtCompileTime, Rhs::ColsAtCompileTime> x(N, b.cols()); - if (!b.allFinite()) { - // A non-finite right-hand side cannot go through the transforms (see - // addProduct): apply the pseudo-inverse -- itself a BCCB operator, - // generated by the 2-D inverse DFT of the thresholded reciprocal symbol -- - // through the direct kernel so Inf/NaN propagate entrywise. - const GeneratorType pg = generatorFromSymbol(sinv); - x.setZero(); - Bccb(pg, ComplexArray(), ComplexArray()).directProduct(x, b.derived(), Scalar(1)); - return x; - } - // The right-hand side is finite here (checked above), so no per-column - // direct kernel can be needed. - applySymbol(x, sinv, b.derived(), Scalar(1), /*accumulate=*/false, - [](Index) { eigen_assert(false && "non-finite column requires a direct kernel"); }); + applySymbol(x, sinv, b.derived(), Scalar(1), /*accumulate=*/false); return x; } /** \returns the numerical rank: the number of symbol entries whose modulus is - * no smaller than the threshold \c N * epsilon * max|symbol|, both evaluated - * in an exactly rescaled frame so the moduli cannot overflow (see - * scaledModuli()) and clamped from below like \c SVDBase::rank(). This is the + * no smaller than the threshold \c N * epsilon * max|symbol|, clamped from + * below by the smallest normal number like \c SVDBase::rank(). This is the * same threshold \ref solve uses to decide which Fourier components to invert, * and the comparison is strict like SVDBase's, so an entry sitting exactly on * the threshold still counts as non-zero. */ Index rank() const { const ComplexArray s = symbol(); - RealArray mods; - RealScalar tol; - scaledModuli(s, mods, tol); - return (!(mods.array() < tol)).count(); // negated so NaN entries count as non-zero + const RealArray mods = s.cwiseAbs(); + return (!(mods.array() < internal::structured_rank_threshold(s, mods))).count(); // NaN entries count as non-zero } /** \returns the inverse of \c *this, itself a \c Bccb operator: the one @@ -406,21 +379,12 @@ /** \internal Computes \c dst += alpha * (*this) * rhs. \c ProductScalar is the * promoted scalar of the product (complex when a real operator is applied to a - * complex right-hand side); the accumulation runs in the promoted type. - * - * Non-finite data takes the direct O(N^2) kernel: the transforms would smear a - * single Inf/NaN into NaNs across the whole output, where the dense product - * only propagates it through the dot products that touch it. A non-finite - * generating array or cached symbol (which can overflow even for a finite - * generator: the 2-D DFT accumulates up to N addends) routes the whole - * product; a non-finite right-hand-side column is detected inside the FFT - * loop -- in the same pass that derives its scaling exponent, so finite data - * pays no extra scan -- and falls back per column. */ + * complex right-hand side); the accumulation runs in the promoted type. */ template <typename Dest, typename Rhs, typename ProductScalar> void addProduct(Dest& dst, const Rhs& rhs, const ProductScalar& alpha) const { const Index N = rows(); eigen_assert(rhs.rows() == N && "invalid product: dimensions do not match"); - if (N <= internal::structured_direct_threshold() || !m_fftUsable) { + if (N <= internal::structured_direct_threshold()) { directProduct(dst, rhs, alpha); return; } @@ -428,7 +392,7 @@ // 5-smooth (see computeProdSymbol()); the exact-size transform of such a // dimension runs through kissfft's quadratic generic butterfly. const ComplexArray& s = m_prodSymbol.size() > 0 ? m_prodSymbol : m_symbol; - applySymbol(dst, s, rhs, alpha, /*accumulate=*/true, [&](Index k) { directProductColumn(dst, rhs, k, alpha); }); + applySymbol(dst, s, rhs, alpha, /*accumulate=*/true); } private: @@ -464,19 +428,7 @@ * adjoint() and inverse(), whose symbols are cheap transformations of the * existing ones. */ Bccb(const GeneratorType& g, const ComplexArray& symbol, const ComplexArray& prodSymbol) - : m_g(g), m_symbol(symbol), m_prodSymbol(prodSymbol) { - m_fftUsable = computeFftUsable(); - } - - /** \internal Whether products may take the FFT path: the generating array and - * the cached symbols must be finite. The symbols accumulate up to N - * (respectively p2*p1) addends, so they can overflow to Inf even for a finite - * generator; such operators fall back to the direct kernel, which stays - * exact. */ - bool computeFftUsable() const { - return m_g.allFinite() && (m_symbol.size() == 0 || m_symbol.allFinite()) && - (m_prodSymbol.size() == 0 || m_prodSymbol.allFinite()); - } + : m_g(g), m_symbol(symbol), m_prodSymbol(prodSymbol) {} /** \internal \returns the symbol products should use for a generating array * \a g: empty when both block dimensions are 5-smooth (the exact-size @@ -515,9 +467,7 @@ * computes \c dst.col(k) += alpha * (*this) * rhs.col(k) without transforms, * as a plain scalar loop (the two-level segment-based middle tier of the 1-D * operators does not pay off here: the within-block segments are too short at - * these sizes). Serves operators below the FFT threshold and any column - * involving non-finite data, whose entrywise IEEE semantics the transforms - * cannot preserve. */ + * these sizes), for operators below the FFT threshold. */ template <typename Dest, typename Rhs, typename ProductScalar> void directProductColumn(Dest& dst, const Rhs& rhs, Index k, const ProductScalar& alpha) const { const Index N = rows(); @@ -545,108 +495,39 @@ * \a s itself: for the padded product symbol (see computeProdSymbol()) the * column is zero-padded into the (p2, p1) grid and the leading (n2, n1) block * of the back-transform is read out; for an exact-size symbol the grid is - * (n2, n1) and no padding is involved. - * - * The transforms sum up to \c N inputs, so intermediates can overflow even - * when every entry of the true result is representable. To keep the whole - * pipeline overflow-free for finite inputs anywhere in the floating-point - * range, the symbol and each right-hand-side column are rescaled by an exact - * power of two that brings their maximum modulus below one -- huge generators - * make huge symbols, hence both sides -- and the removed exponent is folded - * back into the output through per-entry ldexp, which saturates cleanly to - * zero or infinity if the true result itself leaves the representable range. - * The exponents come from the component-wise magnitudes: the modulus of a - * finite complex value near the overflow threshold is not representable, - * which would silently disable the scaling exactly where it is needed. - * - * A single plain (fast-max) reduction pass per column yields the finiteness - * routing, the zero-column shortcut and the scaling exponent (see - * internal::structured_exponent_bound_finite() for why missing a NaN in the - * fast max cannot change a routing decision). Non-finite columns are handed - * to \a directColumn, the caller's per-column direct kernel, so the remaining - * columns keep the fast path; a non-finite symbol is the caller's - * responsibility (addProduct routes it to the direct kernel up front, and - * solve() takes its dedicated pseudo-inverse fallback for non-finite - * right-hand sides). Zero columns short-circuit to zero, but only under a - * finite symbol (Inf*0 and NaN*0 are NaN entrywise) and after an exact - * recheck: the fast max can miss a NaN hiding among zeros (an Inf always - * surfaces), and such a column must fall back, not shortcut. */ - template <typename Dest, typename Rhs, typename ProductScalar, typename DirectColumn> - void applySymbol(Dest& dst, const ComplexArray& s, const Rhs& rhs, const ProductScalar& alpha, bool accumulate, - DirectColumn&& directColumn) const { - const Index n2 = blockSize(), n1 = numBlocks(), N = rows(); + * (n2, n1) and no padding is involved. As in internal::structured_fft_apply(), + * the transforms run in plain floating-point arithmetic. */ + template <typename Dest, typename Rhs, typename ProductScalar> + void applySymbol(Dest& dst, const ComplexArray& s, const Rhs& rhs, const ProductScalar& alpha, + bool accumulate) const { + const Index n2 = blockSize(), n1 = numBlocks(); const Index p2 = s.rows(), p1 = s.cols(); // the symbol's transform grid: (n2, n1) or the padded product grid const bool padded = p2 != n2 || p1 != n1; - const bool sFinite = s.allFinite(); - // max|s| < 2^es; component bounds avoid overflow in complex moduli. - const int es = internal::structured_exponent_bound(s); - ComplexArray sScaled; - if (es != 0) { - sScaled = s; - ldexpInPlace(sScaled, -es); - } - const ComplexArray& sUse = es != 0 ? sScaled : s; - Matrix<ProductScalar, Dynamic, 1> xc(N); + ComplexVector xc(rows()); ComplexArray X(p2, p1), Xn; // Xn: the leading-block extraction of a padded grid for (Index k = 0; k < rhs.cols(); ++k) { - xc = rhs.col(k).template cast<ProductScalar>(); - const RealScalar m = xc.realView().cwiseAbs().maxCoeff(); - if (!(numext::isfinite)(m)) { - directColumn(k); - continue; - } - if (m == RealScalar(0)) { - // The fast max cannot hide an Inf (those comparisons are ordered), but - // it can miss a NaN among zeros: recheck exactly before shortcutting. - if ((xc.array() == ProductScalar(0)).all()) { - if (sFinite) { - // An exactly zero column maps to an exactly zero column -- unless - // the symbol holds Inf or NaN, whose products with zero are NaN; - // falling through to the transforms produces exactly that. - if (!accumulate) dst.col(k).setZero(); - continue; - } - } else { - directColumn(k); - continue; - } - } - int ex = 0; // stays 0 for an all-zero column: no scaling - if (m > RealScalar(0)) { - EIGEN_USING_STD(frexp); - frexp(m, &ex); - EIGEN_IF_CONSTEXPR (NumTraits<ProductScalar>::IsComplex) ++ex; - } // reshaped() defaults to column-major traversal, the flattening the // two-level structure keys on, regardless of EIGEN_DEFAULT_TO_ROW_MAJOR. + // Plain objects keep both reshapes linear. + xc = rhs.col(k).template cast<Complex>(); if (padded) { X.setZero(); - X.topLeftCorner(n2, n1) = xc.reshaped(n2, n1).template cast<Complex>(); + X.topLeftCorner(n2, n1) = xc.reshaped(n2, n1); } else { - X = xc.reshaped(n2, n1).template cast<Complex>(); + X = xc.reshaped(n2, n1); } - ldexpInPlace(X, -ex); fft2(X); - X.array() *= sUse.array(); + X.array() *= s.array(); ifft2(X); if (padded) Xn = X.topLeftCorner(n2, n1); - ComplexArray& out = padded ? Xn : X; - ldexpInPlace(out, ex + es); + const auto out = internal::structured_scalar_part_impl<ProductScalar>::run((padded ? Xn : X).reshaped()); if (accumulate) - dst.col(k) += alpha * internal::structured_scalar_part_impl<ProductScalar>::run(out.reshaped()); + dst.col(k) += alpha * out; else - dst.col(k) = alpha * internal::structured_scalar_part_impl<ProductScalar>::run(out.reshaped()); + dst.col(k) = alpha * out; } } - /** \internal Multiplies every entry of \a X by 2^e, exactly. The per-entry - * ldexp saturates to zero / infinity component-wise without ever forming the - * (possibly unrepresentable) scale factor 2^e itself. */ - static void ldexpInPlace(ComplexArray& X, int e) { - if (e == 0) return; - X.realView() = X.realView().array().ldexp(e).matrix(); - } - /** \internal In-place forward or inverse 2-D FFT by the row-column algorithm: * transform every column (length n2), then every row (length n1). Length-1 * transforms are the identity (and unsupported by the kissfft backend), hence @@ -718,32 +599,6 @@ return R; } - /** \internal Computes the moduli of the symbol entries and the matching - * rank/pseudo-inversion threshold, both evaluated in an exactly rescaled frame - * [5]: the entries are pre-scaled by a power of two chosen so no modulus can - * overflow. A finite complex entry near the overflow threshold has a - * non-representable modulus, which would otherwise turn the threshold into - * infinity and misclassify every other entry (rank under-reported, solve() - * zeroing valid Fourier modes). The rescaling is exact, so comparing scaled - * moduli against the scaled threshold is equivalent to the unscaled - * comparison. The threshold keeps the N * epsilon * max|s| convention of [3], - * chapter 5.4, and the smallest-normal clamp -- carried into the scaled frame, - * where its underflowing to zero for a huge frame is correct: no entry of such - * a symbol can sit below the smallest normal number. Entries at or above the - * threshold, in particular a smallest-normal entry of a moderate symbol, are - * inverted (their reciprocals are finite). */ - static void scaledModuli(const ComplexArray& s, RealArray& mods, RealScalar& tol) { - const int e = numext::maxi(internal::structured_exponent_bound(s), 0); - // Two exact factors, as in Circulant::scaledModuli(): a single 2^-e is itself - // subnormal once the frame exceeds the exponent range, and reads as zero under - // flush-to-zero -- collapsing every scaled modulus, and the threshold with - // them. Neither factor exceeds one, so no intermediate underflows on its own. - const RealScalar down1 = numext::ldexp(RealScalar(1), -(e / 2)), down2 = numext::ldexp(RealScalar(1), -(e - e / 2)); - mods = ((s * down1) * down2).cwiseAbs(); - tol = numext::maxi(RealScalar(s.size()) * NumTraits<RealScalar>::epsilon() * mods.maxCoeff(), - ((std::numeric_limits<RealScalar>::min)() * down1) * down2); - } - /** \internal Writes the unit-norm 2-D Fourier eigenvector of frequencies * \c (f1, f2) into column \a dstCol of \a F: entry \c b1*n2 + i2 is * exp(2 pi i (b1 f1 / n1 + i2 f2 / n2)) / sqrt(N). All frequency products are @@ -777,7 +632,6 @@ // The padded embedding symbol products use when a block dimension is not // 5-smooth; empty otherwise. See computeProdSymbol(). ComplexArray m_prodSymbol; - bool m_fftUsable; }; /** \ingroup StructuredMatrices_Module
diff --git a/contrib/Eigen/src/StructuredMatrices/Cauchy.h b/contrib/Eigen/src/StructuredMatrices/Cauchy.h index e5df904..9d10bb1 100644 --- a/contrib/Eigen/src/StructuredMatrices/Cauchy.h +++ b/contrib/Eigen/src/StructuredMatrices/Cauchy.h
@@ -14,7 +14,7 @@ // (fraction, exponent) pair to avoid spurious overflow/underflow. // [2] P. H. Sterbenz, "Floating-Point Computation", Prentice-Hall, 1974. // Scaling by a power of two is exact, the property the balanced -// accumulation and the guarded boundary-node evaluations rely on. +// accumulation relies on. #ifndef EIGEN_STRUCTURED_CAUCHY_H #define EIGEN_STRUCTURED_CAUCHY_H @@ -65,62 +65,6 @@ static constexpr int CoeffReadCost = Dynamic; }; -/** \internal \returns the Cauchy coefficient \c 1 / (a - b), guarded against a - * spurious overflow of the difference (internal::structured_guarded_diff): a - * naively formed coefficient would collapse to 1/Inf = 0, where the true value - * is a representable (possibly subnormal) number. When the guard fires, one - * half is divided directly by the halved difference, so the subnormal result is - * rounded only once. Every coefficient evaluation exposed by Cauchy -- coeff(), - * dense materialization and the product kernel -- goes through this helper. - * CauchyLU and determinant() use the same guarded difference inside their - * balanced accumulations. */ -template <typename Scalar> -Scalar cauchy_reciprocal_diff(const Scalar& a, const Scalar& b) { - using RealScalar = typename NumTraits<Scalar>::Real; - int e; - const Scalar t = structured_guarded_diff(a, b, e); - return Scalar(e == 0 ? RealScalar(1) : RealScalar(0.5)) / t; -} - -/** \internal \returns a guarded node difference as a balanced mantissa, with - * its power of two in \a exponent. */ -template <typename Scalar> -Scalar cauchy_balanced_diff(const Scalar& a, const Scalar& b, Index& exponent) { - int guardedExponent; - const Scalar difference = structured_guarded_diff(a, b, guardedExponent); - exponent = guardedExponent; - return structured_balance(difference, exponent); -} - -/** \internal Evaluates a GKO Schur-complement entry from generators stored as - * mantissa/exponent pairs. The guarded node difference is balanced before the - * division, and the accumulated power of two is applied only to the final - * result, so finite entries do not overflow or underflow in intermediate - * generator arithmetic. */ -template <typename Scalar> -Scalar cauchy_scaled_entry(const Scalar& a, Index aExponent, const Scalar& b, Index bExponent, const Scalar& x, - const Scalar& y) { - // Preserve the single-rounding coefficient path for the initial generators. - if (aExponent == 0 && bExponent == 0 && a == Scalar(1) && b == Scalar(1)) return cauchy_reciprocal_diff(x, y); - Index diffExponent; - const Scalar diff = cauchy_balanced_diff(x, y, diffExponent); - Index exponent = aExponent + bExponent - diffExponent; - const Scalar value = structured_balance(Scalar((a * b) / diff), exponent); - return structured_ldexp_clamped(value, exponent); -} - -/** \internal \returns the balanced mantissa of \c (a - b) / (a - c), with its - * power of two in \a exponent. Separately balancing both guarded differences - * prevents a representable ratio from passing through Inf/Inf or zero. */ -template <typename Scalar> -Scalar cauchy_balanced_diff_ratio(const Scalar& a, const Scalar& b, const Scalar& c, Index& exponent) { - Index numeratorExponent, denominatorExponent; - const Scalar numerator = cauchy_balanced_diff(a, b, numeratorExponent); - const Scalar denominator = cauchy_balanced_diff(a, c, denominatorExponent); - exponent = numeratorExponent - denominatorExponent; - return structured_balance(Scalar(numerator / denominator), exponent); -} - } // namespace internal /** \ingroup StructuredMatrices_Module @@ -191,7 +135,6 @@ EIGEN_STATIC_ASSERT_VECTOR_ONLY(XDerived) EIGEN_STATIC_ASSERT_VECTOR_ONLY(YDerived) eigen_assert(m_x.size() > 0 && m_y.size() > 0 && "Cauchy node vectors must be non-empty"); - m_boundary = computeBoundary(); } EIGEN_DEVICE_FUNC Index rows() const { return m_x.size(); } @@ -202,11 +145,8 @@ /** \returns the column node vector \c y. */ const ColNodeVector& colNodes() const { return m_y; } - /** \returns the coefficient at row \a row and column \a col, evaluated - * through the guarded reciprocal: nodes at the overflow boundary yield the - * correctly rounded (possibly subnormal) value instead of a spurious - * 1/Inf = 0. */ - Scalar coeff(Index row, Index col) const { return internal::cauchy_reciprocal_diff(m_x.coeff(row), m_y.coeff(col)); } + /** \returns the coefficient at row \a row and column \a col. */ + Scalar coeff(Index row, Index col) const { return Scalar(1) / (m_x.coeff(row) - m_y.coeff(col)); } /** \returns the transpose of \c *this, itself a Cauchy operator: * \f$ C(x,y)^T = C(-y,-x) \f$. */ @@ -229,14 +169,11 @@ * convention of LINPACK's xGEDI [1]) -- every factor and the running value * are renormalized to unit magnitude with the power of two tracked separately * (exact frexp/ldexp rescaling [2]), numerator factors multiplied in and - * denominator factors divided out -- so no intermediate can overflow or - * underflow when the determinant itself is representable. A node difference - * that overflows even though both nodes are finite (nodes near opposite ends - * of the exponent range) is recomputed from the halved nodes -- exact, since - * such an overflow implies huge normal operands -- with the removed power of - * two entering the same exponent bookkeeping, so it too cannot push the - * accumulation to a spurious Inf. Zero factors (coincident \c x or \c y - * nodes) and genuinely non-finite factors propagate exactly. */ + * denominator factors divided out -- so no partial product can overflow or + * underflow when the determinant itself is representable. Zero factors + * (coincident \c x or \c y nodes) and non-finite factors propagate exactly; + * a node difference that overflows makes its factor infinite, which turns the + * determinant into NaN when it meets a zero or another infinite factor. */ Scalar determinant() const { eigen_assert(rows() == cols() && "Cauchy::determinant requires a square matrix"); const Index n = rows(); @@ -244,18 +181,15 @@ Index exponent = 0; for (Index j = 1; j < n; ++j) for (Index i = 0; i < j; ++i) { - Index factorExponent; - const Scalar xDiff = internal::cauchy_balanced_diff(m_x.coeff(j), m_x.coeff(i), factorExponent); - exponent += factorExponent; + const Scalar xDiff = internal::structured_balance(Scalar(m_x.coeff(j) - m_x.coeff(i)), exponent); det = internal::structured_balance(Scalar(det * xDiff), exponent); - const Scalar yDiff = internal::cauchy_balanced_diff(m_y.coeff(i), m_y.coeff(j), factorExponent); - exponent += factorExponent; + const Scalar yDiff = internal::structured_balance(Scalar(m_y.coeff(i) - m_y.coeff(j)), exponent); det = internal::structured_balance(Scalar(det * yDiff), exponent); } for (Index j = 0; j < n; ++j) for (Index i = 0; i < n; ++i) { Index denomExponent = 0; - const Scalar d = internal::cauchy_balanced_diff(m_x.coeff(i), m_y.coeff(j), denomExponent); + const Scalar d = internal::structured_balance(Scalar(m_x.coeff(i) - m_y.coeff(j)), denomExponent); exponent -= denomExponent; det = internal::structured_balance(Scalar(det / d), exponent); } @@ -263,10 +197,7 @@ } /** \internal Writes the dense representation into \a dst, one vectorized - * column at a time. Operators whose nodes reach the overflow boundary (see - * computeBoundary()) instead evaluate every entry through the guarded - * reciprocal, staying consistent with coeff(). Invoked through - * \c dense = cauchy; */ + * column at a time. Invoked through \c dense = cauchy; */ template <typename Dest> void evalTo(Dest& dst) const { applyAssignment(dst, internal::assign_op<typename Dest::Scalar, Scalar>()); @@ -285,10 +216,10 @@ } /** \returns the product expression \c (*this) * \a v, evaluated directly at - * O(mn) operations and O(1) extra storage. The expression carries the default - * product tag, so assigning it behaves like any dense product: a temporary - * resolves aliasing between the destination and \a v, and \c .noalias() skips - * it. */ + * O(mn) operations per right-hand side and at most O(m) extra storage. The + * expression carries the default product tag, so assigning it behaves like any + * dense product: a temporary resolves aliasing between the destination and + * \a v, and \c .noalias() skips it. */ template <typename Rhs> Product<Cauchy, Rhs> operator*(const MatrixBase<Rhs>& v) const { EIGEN_STATIC_ASSERT(ColsAtCompileTime == Dynamic || Rhs::RowsAtCompileTime == Dynamic || @@ -300,60 +231,54 @@ /** \internal Computes \c dst += alpha * (*this) * rhs. \c ProductScalar is the * promoted scalar of the product (complex when a real operator is applied to a - * complex right-hand side); the accumulation runs in the promoted type. The - * coefficients enter through the guarded reciprocal, so the product uses - * exactly the values coeff() exposes, boundary nodes included. */ + * complex right-hand side); the accumulation runs in the promoted type. Each + * column of the operator is formed on the fly, vectorized over the rows; for + * several right-hand sides of the operator's own scalar type it is formed once + * per block of rows and reused across them, with identical results. */ template <typename Dest, typename Rhs, typename ProductScalar> void addProduct(Dest& dst, const Rhs& rhs, const ProductScalar& alpha) const { - const Index m = rows(), n = cols(); + const Index n = cols(); eigen_assert(rhs.rows() == n && "invalid product: dimensions do not match"); - for (Index k = 0; k < rhs.cols(); ++k) - for (Index i = 0; i < m; ++i) { - const Scalar xi = m_x.coeff(i); - ProductScalar acc(0); - for (Index j = 0; j < n; ++j) acc += rhs.coeff(j, k) * internal::cauchy_reciprocal_diff(xi, m_y.coeff(j)); - dst.coeffRef(i, k) += alpha * acc; + // A unit alpha must not multiply: even the identity complex scalar (1,0) + // pollutes an (Inf,0) value with NaN through the 0*Inf cross term. + const bool unitAlpha = alpha == ProductScalar(1); + const auto weight = [&](Index j, Index k) { + return unitAlpha ? ProductScalar(rhs.coeff(j, k)) : ProductScalar(alpha * rhs.coeff(j, k)); + }; + // Reuse saves (r-1)/r of the divisions. A real operator applied to a complex + // right-hand side gains nothing: the complex updates dominate its real divisions. + if (rhs.cols() > 1 && std::is_same<ProductScalar, Scalar>::value) { + // Row blocks keep the r destination columns of a block in cache across all + // n reciprocal columns; the floor keeps the per-column updates long. + constexpr Index kBlockBytes = Index(1) << 19; + constexpr Index kMinBlockRows = 1024; + const Index m = rows(), r = rhs.cols(); + const Index blockRows = numext::mini(m, numext::maxi(kMinBlockRows, kBlockBytes / (r * Index(sizeof(Scalar))))); + Matrix<Scalar, Dynamic, 1, ColMajor, Rows_> reciprocals(blockRows); + for (Index i = 0; i < m; i += blockRows) { + const Index b = numext::mini(blockRows, m - i); + for (Index j = 0; j < n; ++j) { + reciprocals.head(b) = (m_x.segment(i, b).array() - m_y.coeff(j)).inverse(); + for (Index k = 0; k < r; ++k) dst.col(k).segment(i, b) += weight(j, k) * reciprocals.head(b); + } } + return; + } + for (Index k = 0; k < rhs.cols(); ++k) + for (Index j = 0; j < n; ++j) dst.col(k) += weight(j, k) * (m_x.array() - m_y.coeff(j)).inverse().matrix(); } private: template <typename Dest, typename Assignment> void applyAssignment(Dest& dst, const Assignment& assignment) const { - if (m_boundary) { - for (Index j = 0; j < cols(); ++j) - for (Index i = 0; i < rows(); ++i) - assignment.assignCoeff(dst.coeffRef(i, j), internal::cauchy_reciprocal_diff(m_x.coeff(i), m_y.coeff(j))); - return; - } for (Index j = 0; j < cols(); ++j) { auto dstColumn = dst.col(j); internal::call_assignment_no_alias(dstColumn, (m_x.array() - m_y.coeff(j)).inverse().matrix(), assignment); } } - /** \internal Whether some node difference \c x_i - y_j could overflow on - * finite nodes (conservative componentwise bound: the largest \c |x| and - * \c |y| components sum past the largest finite value), or a node is - * non-finite. The dense materialization then takes the guarded scalar path - * instead of the vectorized column expression; for moderate nodes the bound - * guarantees the two paths are bit-identical, so the vectorized path is kept. - * The magnitudes are taken componentwise: the modulus of a finite complex - * node near the overflow threshold is not representable. */ - bool computeBoundary() const { - RealScalar mx, my; - EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) { - mx = numext::maxi(m_x.real().cwiseAbs().maxCoeff(), m_x.imag().cwiseAbs().maxCoeff()); - my = numext::maxi(m_y.real().cwiseAbs().maxCoeff(), m_y.imag().cwiseAbs().maxCoeff()); - } else { - mx = m_x.cwiseAbs().maxCoeff(); - my = m_y.cwiseAbs().maxCoeff(); - } - return !(mx + my <= (std::numeric_limits<RealScalar>::max)()); - } - RowNodeVector m_x; ColNodeVector m_y; - bool m_boundary; }; /** \ingroup StructuredMatrices_Module @@ -407,7 +332,6 @@ EIGEN_GENERIC_PUBLIC_INTERFACE(CauchyLU) using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic>; using DenseVector = Matrix<Scalar, Dynamic, 1>; - using IndexVector = Matrix<Index, Dynamic, 1>; /** Default constructor; call \ref compute before \ref solve. */ CauchyLU() : m_isInitialized(false), m_info(InvalidInput) {} @@ -433,19 +357,15 @@ const DenseVector y = C.colNodes(); DenseVector a = DenseVector::Ones(n); DenseVector b = DenseVector::Ones(n); - IndexVector aExponent = IndexVector::Zero(n); - IndexVector bExponent = IndexVector::Zero(n); m_lu.resize(n, n); m_perm.resize(static_cast<std::size_t>(n)); m_info = Success; for (Index k = 0; k < n; ++k) { - // Guarded differences preserve subnormal Schur-complement entries at the - // exponent boundary instead of creating zero pivots or NaN generators. Index piv = k; RealScalar best(-1); for (Index i = k; i < n; ++i) { - m_lu(i, k) = internal::cauchy_scaled_entry(a[i], aExponent[i], b[k], bExponent[k], x[i], y[k]); + m_lu(i, k) = a[i] * b[k] / (x[i] - y[k]); const RealScalar mag = numext::abs(m_lu(i, k)); if ((numext::isnan)(mag) || mag > best) { best = mag; @@ -456,7 +376,6 @@ m_lu.row(piv).head(k + 1).swap(m_lu.row(k).head(k + 1)); std::swap(x[piv], x[k]); std::swap(a[piv], a[k]); - std::swap(aExponent[piv], aExponent[k]); } m_perm[static_cast<std::size_t>(k)] = piv; const Scalar pivot = m_lu(k, k); @@ -468,21 +387,11 @@ } m_lu.col(k).tail(n - k - 1) /= pivot; for (Index j = k + 1; j < n; ++j) { - m_lu(k, j) = internal::cauchy_scaled_entry(a[k], aExponent[k], b[j], bExponent[j], x[k], y[j]); + m_lu(k, j) = a[k] * b[j] / (x[k] - y[j]); if (!(numext::isfinite)(m_lu(k, j))) m_info = NumericalIssue; } - for (Index i = k + 1; i < n; ++i) { - Index ratioExponent; - const Scalar ratio = internal::cauchy_balanced_diff_ratio(x[i], x[k], y[k], ratioExponent); - aExponent[i] += ratioExponent; - a[i] = internal::structured_balance(Scalar(a[i] * ratio), aExponent[i]); - } - for (Index j = k + 1; j < n; ++j) { - Index ratioExponent; - const Scalar ratio = internal::cauchy_balanced_diff_ratio(y[j], y[k], x[k], ratioExponent); - bExponent[j] += ratioExponent; - b[j] = internal::structured_balance(Scalar(b[j] * ratio), bExponent[j]); - } + for (Index i = k + 1; i < n; ++i) a[i] *= (x[i] - x[k]) / (x[i] - y[k]); + for (Index j = k + 1; j < n; ++j) b[j] *= (y[j] - y[k]) / (y[j] - x[k]); } m_isInitialized = true; return *this;
diff --git a/contrib/Eigen/src/StructuredMatrices/Circulant.h b/contrib/Eigen/src/StructuredMatrices/Circulant.h index ee66df2..e45ab14 100644 --- a/contrib/Eigen/src/StructuredMatrices/Circulant.h +++ b/contrib/Eigen/src/StructuredMatrices/Circulant.h
@@ -23,8 +23,7 @@ // (fraction, exponent) pair to avoid spurious overflow/underflow. // [5] P. H. Sterbenz, "Floating-Point Computation", Prentice-Hall, 1974. // Scaling by a power of two is exact, the property the balanced -// accumulation, the rescaled rank threshold and the scaled FFT products -// rely on. +// accumulation relies on. #ifndef EIGEN_STRUCTURED_CIRCULANT_H #define EIGEN_STRUCTURED_CIRCULANT_H @@ -131,7 +130,6 @@ m_symbol = computeSymbol(); m_prodSymbol = computeProdSymbol(m_col); } - m_fftUsable = computeFftUsable(); } EIGEN_DEVICE_FUNC Index rows() const { return m_col.size(); } @@ -266,69 +264,24 @@ const Index n = rows(); eigen_assert(b.rows() == n && "right-hand side has the wrong number of rows"); const ComplexVector s = symbol(); - RealVector mods; - RealScalar tol; - int es = 0; - scaledModuli(s, mods, tol, &es); - // Strictly-below-threshold entries are zeroed, matching SVDBase::rank(), so a - // smallest-normal 1x1 operator stays invertible. NaN moduli fail the - // comparison and land in the inverted set, so a NaN input propagates to the - // output instead of being silently zeroed. - ComplexVector sinv(n); - internal::structured_symbol_reciprocal(s.data(), mods.data(), tol, sinv.data(), n); + const RealVector mods = s.cwiseAbs(); + const ComplexVector sinv = internal::structured_pinv_symbol(s, mods, internal::structured_rank_threshold(s, mods)); Matrix<Scalar, Size_, Rhs::ColsAtCompileTime> x(n, b.cols()); x.setZero(); - if (!b.allFinite()) { - // A non-finite right-hand side cannot go through the transforms (see - // addProduct): apply the pseudo-inverse -- itself a circulant matrix, - // generated by the inverse DFT of the thresholded reciprocal symbol -- - // through the direct kernel so Inf/NaN propagate entrywise. - GeneratorType pcol(n); - if (n == 1) { - pcol[0] = internal::structured_scalar_part_impl<Scalar>::run_scalar(sinv.coeff(0)); - } else { - auto&& fft = internal::structured_fft_engine<RealScalar>(); - ComplexVector pt(n); - fft.inv(pt, sinv, n); - pcol = internal::structured_scalar_part_impl<Scalar>::run(pt); - } - Circulant(pcol, ComplexVector(), ComplexVector()).directProduct(x, b.derived(), Scalar(1)); - return x; - } - // The reciprocal of the largest symbol entry is about 2^-es: subnormal, hence - // lost under flush-to-zero, only for a symbol at the top of the exponent range. - // Dividing by the symbol keeps that mode normal but costs a complex division - // per entry, so it is reserved for that band. - constexpr Index kSubnormalReciprocal = -(Index(std::numeric_limits<RealScalar>::min_exponent) - 1) - 2; - if (es <= kSubnormalReciprocal) { - internal::structured_fft_apply(x, sinv, n, b.derived(), Scalar(1)); - return x; - } - // ldexp per component rather than a multiply by 2^-es: for a near-boundary - // symbol that constant is itself subnormal, and reads as zero under - // flush-to-zero. The apply folds es back into the output scaling. - const ComplexVector sb = s.unaryExpr([es](const Complex& z) { return internal::structured_ldexp_clamped(z, -es); }); - // mods and tol are already in a balanced frame, so rebalancing the symbol - // leaves the threshold comparison unchanged. - internal::structured_symbol_divide<RealVector, RealScalar> divide{&mods, tol}; - internal::structured_fft_apply( - x, sb, n, b.derived(), Scalar(1), - [](Index) { eigen_assert(false && "non-finite column requires a direct kernel"); }, divide, es); + internal::structured_fft_apply(x, sinv, n, b.derived(), Scalar(1)); return x; } /** \returns the numerical rank: the number of symbol entries whose modulus is - * no smaller than the threshold \c n * epsilon * max_k|symbol[k]|, both - * evaluated in an exactly rescaled frame so the moduli cannot overflow (see - * scaledModuli()). This is the same threshold \ref solve uses to decide which - * Fourier components to invert, and the comparison is strict like SVDBase's, - * so an entry sitting exactly on the threshold still counts as non-zero. */ + * no smaller than the threshold \c n * epsilon * max_k|symbol[k]|, clamped from + * below by the smallest normal number. This is the same threshold \ref solve + * uses to decide which Fourier components to invert, and the comparison is + * strict like SVDBase's, so an entry sitting exactly on the threshold still + * counts as non-zero. */ Index rank() const { const ComplexVector s = symbol(); - RealVector mods; - RealScalar tol; - scaledModuli(s, mods, tol); - return (!(mods.array() < tol)).count(); // negated so NaN entries count as non-zero + const RealVector mods = s.cwiseAbs(); + return (!(mods.array() < internal::structured_rank_threshold(s, mods))).count(); // NaN entries count as non-zero } /** \returns the inverse of \c *this, itself a \c Circulant operator: the one @@ -337,8 +290,7 @@ * pseudo-inverse solve of a rank-deficient operator. */ Circulant inverse() const { const Index n = rows(); - ComplexVector sinv = symbol(); - internal::structured_symbol_reciprocal(sinv.data(), n); + const ComplexVector sinv = symbol().cwiseInverse(); GeneratorType col(n); if (n == 1) { col = internal::structured_scalar_part_impl<Scalar>::run(sinv); @@ -435,20 +387,12 @@ /** \internal Computes \c dst += alpha * (*this) * rhs. \c ProductScalar is the * promoted scalar of the product (complex when a real operator is applied to a - * complex right-hand side); the accumulation runs in the promoted type. - * - * Non-finite data takes the direct O(n^2) kernel: the transforms would smear a - * single Inf/NaN into NaNs across the whole output, where the dense product - * only propagates it through the dot products that touch it. A non-finite - * generator or cached symbol (which can overflow even for a finite generator) - * routes the whole product; a non-finite right-hand-side column is detected - * inside the FFT loop -- in the same pass that derives its scaling exponent, - * so finite data pays no extra scan -- and falls back per column. */ + * complex right-hand side); the accumulation runs in the promoted type. */ template <typename Dest, typename Rhs, typename ProductScalar> void addProduct(Dest& dst, const Rhs& rhs, const ProductScalar& alpha) const { const Index n = rows(); eigen_assert(rhs.rows() == n && "invalid product: dimensions do not match"); - if (n <= internal::structured_direct_threshold() || !m_fftUsable) { + if (n <= internal::structured_direct_threshold()) { directProduct(dst, rhs, alpha); return; } @@ -456,7 +400,7 @@ // 5-smooth (see computeProdSymbol()); the exact-size transform of such a // size runs through kissfft's quadratic generic butterfly. const ComplexVector& s = m_prodSymbol.size() > 0 ? m_prodSymbol : m_symbol; - internal::structured_fft_apply(dst, s, n, rhs, alpha, [&](Index k) { directProductColumn(dst, rhs, k, alpha); }); + internal::structured_fft_apply(dst, s, n, rhs, alpha); } private: @@ -466,18 +410,7 @@ * constructor. Used by transpose(), conjugate(), adjoint() and inverse(), * whose symbols are cheap transformations of the existing ones. */ Circulant(const GeneratorType& col, const ComplexVector& symbol, const ComplexVector& prodSymbol) - : m_col(col), m_symbol(symbol), m_prodSymbol(prodSymbol) { - m_fftUsable = computeFftUsable(); - } - - /** \internal Whether products may take the FFT path: the generator and the - * cached symbols must be finite. The symbols accumulate up to n (respectively - * p) addends, so they can overflow to Inf even for a finite generator; such - * operators fall back to the direct kernel, which stays exact. */ - bool computeFftUsable() const { - return m_col.allFinite() && (m_symbol.size() == 0 || m_symbol.allFinite()) && - (m_prodSymbol.size() == 0 || m_prodSymbol.allFinite()); - } + : m_col(col), m_symbol(symbol), m_prodSymbol(prodSymbol) {} /** \internal \returns the symbol products should use for a generator \a col of * size n: empty when n is 5-smooth (the exact-size transform is already fast), @@ -503,10 +436,8 @@ } /** \internal Direct O(n^2) kernel for column \a k of the right-hand side: - * computes \c dst.col(k) += alpha * (*this) * rhs.col(k) without transforms. - * Serves operators below the FFT threshold and any column involving - * non-finite data, whose entrywise IEEE semantics the transforms cannot - * preserve. */ + * computes \c dst.col(k) += alpha * (*this) * rhs.col(k) without transforms, + * for operators below the FFT threshold. */ template <typename Dest, typename Rhs, typename ProductScalar> void directProductColumn(Dest& dst, const Rhs& rhs, Index k, const ProductScalar& alpha) const { const Index n = rows(); @@ -540,33 +471,6 @@ for (Index k = 0; k < rhs.cols(); ++k) directProductColumn(dst, rhs, k, alpha); } - /** \internal Computes the moduli of the symbol entries and the matching - * rank/pseudo-inversion threshold, both evaluated in an exactly rescaled frame: - * the entries are pre-scaled by a power of two chosen so no modulus can - * overflow. A finite complex entry near the overflow threshold has a - * non-representable modulus, which would otherwise turn the threshold into - * infinity and misclassify every other entry (rank under-reported, solve() - * zeroing valid Fourier modes). The rescaling is exact, so comparing scaled - * moduli against the scaled threshold is equivalent to the unscaled - * comparison. The threshold keeps the n * epsilon * max_k|s[k]| convention of - * [3], chapter 5.4, and the smallest-normal clamp -- carried into the scaled - * frame, where its underflowing to zero for a huge frame is correct: no entry - * of such a symbol can sit below the smallest normal number. Entries at or - * above the threshold, in particular a smallest-normal entry of a moderate - * symbol, are inverted (their reciprocals are finite). */ - static void scaledModuli(const ComplexVector& s, RealVector& mods, RealScalar& tol, int* frameExp = nullptr) { - const int e = numext::maxi(internal::structured_exponent_bound(s), 0); - if (frameExp) *frameExp = e; - // Two exact factors, as in structured_fft_apply(): a single 2^-e is itself - // subnormal once the frame exceeds the exponent range, and reads as zero under - // flush-to-zero -- collapsing every scaled modulus, and the threshold with - // them. Neither factor exceeds one, so no intermediate underflows on its own. - const RealScalar down1 = numext::ldexp(RealScalar(1), -(e / 2)), down2 = numext::ldexp(RealScalar(1), -(e - e / 2)); - mods = ((s * down1) * down2).cwiseAbs(); - tol = numext::maxi(RealScalar(s.size()) * NumTraits<RealScalar>::epsilon() * mods.maxCoeff(), - ((std::numeric_limits<RealScalar>::min)() * down1) * down2); - } - /** \internal Computes the symbol \a s and the moduli \a mods that order the * SVD, and \returns the permutation sorting those moduli in decreasing order -- * the single ordering singularValues(), matrixU() and matrixV() agree on. @@ -635,7 +539,6 @@ // The padded embedding symbol products use when the size is not 5-smooth; // empty otherwise. See computeProdSymbol(). ComplexVector m_prodSymbol; - bool m_fftUsable; }; /** \ingroup StructuredMatrices_Module
diff --git a/contrib/Eigen/src/StructuredMatrices/DPR1EigenSolver.h b/contrib/Eigen/src/StructuredMatrices/DPR1EigenSolver.h index 9f1e879..0b966c2 100644 --- a/contrib/Eigen/src/StructuredMatrices/DPR1EigenSolver.h +++ b/contrib/Eigen/src/StructuredMatrices/DPR1EigenSolver.h
@@ -44,15 +44,12 @@ * * Both signs of \f$ \rho \f$ are supported (negative \f$ \rho \f$ is handled by * negating the matrix), as are \f$ \rho = 0 \f$, zero \c z, repeated diagonal - * entries and any ordering of \c d. The problem is rescaled internally by - * exact powers of two chosen from the component exponents alone, so data - * anywhere in the representable range is handled without internal overflow -- - * including inputs for which \f$ \rho \|z\|^2 \f$ alone exceeds the largest - * finite value while every eigenvalue is still representable. \c InvalidInput - * is reported only for non-finite input or a spectrum that is itself not - * representable. On well-scaled data the rescaling changes no bits of the - * result (rounding can occur only at the subnormal boundary, far below the - * deflation backward-error budget). + * entries and any ordering of \c d. The problem is rescaled internally by the + * exact power of two that brings \f$ \max(\|D\|_\infty, |\rho| \|z\|^2) \f$ into + * [1/2, 1), which changes no bits of the result when \f$ \rho \|z\|^2 \f$ and + * the scaled poles are normal. \c InvalidInput is reported for non-finite + * input, for an update \f$ \rho \|z\|^2 \f$ that overflows, and for a computed + * eigenvalue that is not finite. * * \code * DPR1EigenSolver<double> es(d, rho, z); @@ -66,15 +63,9 @@ * - M. Gu and S. C. Eisenstat, "A stable and efficient algorithm for the * rank-one modification of the symmetric eigenproblem," SIAM J. Matrix Anal. * Appl., 15(4):1266-1276, 1994. - * - N. J. Higham, "Accuracy and Stability of Numerical Algorithms", 2nd ed., - * SIAM, 2002, chapter 27. Avoiding spurious overflow by rescaling with - * powers of two: the whole-problem scale chosen from component exponents - * and the (mantissa, exponent) handling of rho*||z||^2 follow this - * technique. * - P. H. Sterbenz, "Floating-Point Computation", Prentice-Hall, 1974. - * Scaling by a power of two is exact, the property the frexp/ldexp - * problem scaling and the full-range ArrayBase::ldexp unscaling of the - * eigenvalues rely on. + * Scaling by a power of two is exact, the property the problem scaling + * relies on. * * \sa class SelfAdjointEigenSolver */ @@ -118,9 +109,9 @@ } /** \returns \c Success if the decomposition succeeded, \c NoConvergence if a - * secular root or the spectrum's finite-range boundary could not be fully - * resolved, \c InvalidInput if the input was non-finite or its spectrum is - * not representable in the scalar type. */ + * secular root could not be fully resolved, \c InvalidInput if the input was + * non-finite, \f$ \rho \|z\|^2 \f$ overflows, or a computed eigenvalue is + * not finite (the eigenvalues are then NaN). */ ComputationInfo info() const { eigen_assert(m_isInitialized && "DPR1EigenSolver is not initialized."); return m_info; @@ -133,47 +124,6 @@ RealScalar c, s; }; - struct DoubleWord { - RealScalar hi, lo; - }; - - enum class SpectrumRange { Representable, ExactBoundary, Overflow, Uncertain }; - - static DoubleWord addDoubleWords(const DoubleWord& x, const DoubleWord& y) { - DoubleWord result; - internal::twosum(x.hi, x.lo, y.hi, y.lo, result.hi, result.lo); - return result; - } - - static DoubleWord multiplyDoubleWords(const DoubleWord& x, const DoubleWord& y) { - DoubleWord result; - internal::twoprod(x.hi, x.lo, y.hi, y.lo, result.hi, result.lo); - return result; - } - - static DoubleWord divideDoubleWords(const DoubleWord& x, const DoubleWord& y) { - DoubleWord quotient; - internal::doubleword_div_fp(x.hi, x.lo, y.hi, quotient.hi, quotient.lo); - const DoubleWord product = multiplyDoubleWords(quotient, y); - const DoubleWord remainder = addDoubleWords(x, DoubleWord{-product.hi, -product.lo}); - DoubleWord correction; - internal::doubleword_div_fp(remainder.hi, remainder.lo, y.hi, correction.hi, correction.lo); - return addDoubleWords(quotient, correction); - } - - static DoubleWord scaleDoubleWord(const DoubleWord& x, int exponent) { - EIGEN_USING_STD(ldexp) - DoubleWord result{ldexp(x.hi, exponent), ldexp(x.lo, exponent)}; - if ((numext::isfinite)(result.hi)) { - DoubleWord normalized; - internal::fast_twosum(result.hi, result.lo, normalized.hi, normalized.lo); - result = normalized; - } - return result; - } - - static SpectrumRange classifySpectrumRange(const VectorType& d, RealScalar rho, const VectorType& z); - /** \internal Evaluates the shifted secular function * g(tau) = 1 + rho * sum_i zeta_i^2 / (delta_i - tau), with delta_i the pole * offsets relative to the chosen shift. */ @@ -189,116 +139,6 @@ }; template <typename RealScalar_> -typename DPR1EigenSolver<RealScalar_>::SpectrumRange DPR1EigenSolver<RealScalar_>::classifySpectrumRange( - const VectorType& d, RealScalar rho, const VectorType& z) { - // For rho >= 0 only the largest eigenvalue can leave the finite range; with M = highest(), the - // matrix determinant lemma gives lambda_max <= M iff rho * sum_i z_i^2 / (M - d_i) <= 1. Evaluated - // from the original data, not the normalized secular problem, which has already rounded rho*||z||^2. - // Double-word arithmetic and its O(u^2) bounds follow Joldes, Muller and Popescu, "Tight and - // rigorous error bounds for basic building blocks of double-word arithmetic", ACM TOMS 44(2), 2017. - if (rho == RealScalar(0)) return SpectrumRange::Representable; - - EIGEN_USING_STD(frexp) - EIGEN_USING_STD(ldexp) - const RealScalar highest = (std::numeric_limits<RealScalar>::max)(); - const RealScalar highestHalf = highest / RealScalar(2); - int rhoExponent = 0; - const RealScalar rhoFraction = frexp(rho, &rhoExponent); - DoubleWord sum{RealScalar(0), RealScalar(0)}; - bool exactSumValid = true; - Index active = 0; - - for (Index i = 0; i < d.size(); ++i) { - if (z[i] == RealScalar(0)) continue; - ++active; - if (d[i] == highest) return SpectrumRange::Overflow; - - int zExponent = 0; - const RealScalar zFraction = frexp(numext::abs(z[i]), &zExponent); - DoubleWord numerator; - internal::twoprod(rhoFraction, zFraction, numerator.hi, numerator.lo); - const bool numeratorExact = numerator.lo == RealScalar(0) || zFraction == RealScalar(0.5); - if (zFraction == RealScalar(0.5)) { - numerator.hi *= zFraction; - numerator.lo *= zFraction; - DoubleWord normalized; - internal::fast_twosum(numerator.hi, numerator.lo, normalized.hi, normalized.lo); - numerator = normalized; - } else if (numerator.lo == RealScalar(0)) { - DoubleWord product; - internal::twoprod(numerator.hi, zFraction, product.hi, product.lo); - numerator = product; - } else { - DoubleWord product; - internal::twoprod(numerator.hi, numerator.lo, zFraction, product.hi, product.lo); - numerator = product; - } - - // Work with (M - d_i)/2, which cannot overflow even when d_i = -M. - const RealScalar dHalf = d[i] / RealScalar(2); - DoubleWord denominator; - internal::twosum(highestHalf, RealScalar(0), -dHalf, RealScalar(0), denominator.hi, denominator.lo); - const bool denominatorExact = d[i] == RealScalar(0) || dHalf * RealScalar(2) == d[i]; - const DoubleWord unscaledDenominator = denominator; - int denominatorExponent = 0; - frexp(unscaledDenominator.hi, &denominatorExponent); - const RealScalar scaledDenominatorHi = ldexp(unscaledDenominator.hi, -denominatorExponent); - const RealScalar scaledDenominatorLo = ldexp(unscaledDenominator.lo, -denominatorExponent); - const bool denominatorScaleExact = ldexp(scaledDenominatorHi, denominatorExponent) == unscaledDenominator.hi && - ldexp(scaledDenominatorLo, denominatorExponent) == unscaledDenominator.lo; - denominator = scaleDoubleWord(unscaledDenominator, -denominatorExponent); - - const int termExponent = rhoExponent + 2 * zExponent - denominatorExponent - 1; - const DoubleWord quotient = divideDoubleWords(numerator, denominator); - const DoubleWord term = scaleDoubleWord(quotient, termExponent); - if (!(numext::isfinite)(term.hi) || term.hi > RealScalar(2)) return SpectrumRange::Overflow; - - const RealScalar previousSum = sum.hi; - sum = addDoubleWords(sum, term); - if (!(numext::isfinite)(sum.hi) || sum.hi > RealScalar(2)) return SpectrumRange::Overflow; - - // Certify an exact endpoint equality only from terms whose division, - // exponent scaling, and scalar accumulation are all proven exact. This is - // deliberately stricter than the O(u^2) sign estimate below: a rounded - // equality must remain Uncertain rather than become a false Success. - if (exactSumValid) { - const RealScalar normalMin = (std::numeric_limits<RealScalar>::min)(); - bool exactTerm = numeratorExact && denominatorExact && denominatorScaleExact && denominator.lo == RealScalar(0) && - quotient.lo == RealScalar(0) && term.lo == RealScalar(0) && term.hi >= normalMin && - ldexp(term.hi, -termExponent) == quotient.hi; - if (exactTerm) { - DoubleWord recoveredNumerator; - internal::twoprod(quotient.hi, denominator.hi, recoveredNumerator.hi, recoveredNumerator.lo); - exactTerm = recoveredNumerator.hi == numerator.hi && recoveredNumerator.lo == numerator.lo; - } - if (exactTerm) { - const RealScalar larger = numext::maxi(previousSum, term.hi); - const RealScalar smaller = numext::mini(previousSum, term.hi); - // The reversible subtraction remains a proof if FTZ erases a - // subnormal low word from the double-word addition. - exactSumValid = sum.lo == RealScalar(0) && sum.hi - larger == smaller; - } else { - exactSumValid = false; - } - } - } - - if (active == 0) return SpectrumRange::Representable; - if (exactSumValid && sum.hi == RealScalar(1)) return SpectrumRange::ExactBoundary; - - const DoubleWord difference = addDoubleWords(sum, DoubleWord{RealScalar(-1), RealScalar(0)}); - const RealScalar estimate = difference.hi + difference.lo; - const RealScalar unitRoundoff = NumTraits<RealScalar>::epsilon() / RealScalar(2); - // The factor 32 covers two products, corrected division, exponent scaling, - // and accumulation per active term. Proven exact endpoint equality is handled - // above rather than widened into the uncertainty interval. - const RealScalar error = RealScalar(32) * RealScalar(active + 1) * unitRoundoff * unitRoundoff; - if (estimate < -error) return SpectrumRange::Representable; - if (estimate > error) return SpectrumRange::Overflow; - return SpectrumRange::Uncertain; -} - -template <typename RealScalar_> DPR1EigenSolver<RealScalar_>& DPR1EigenSolver<RealScalar_>::compute(const VectorType& d, RealScalar rho, const VectorType& z, int options) { const Index n = d.size(); @@ -326,8 +166,7 @@ } const bool negated = rho < RealScalar(0); - VectorType dW = negated ? VectorType(-d) : d; - RealScalar rhoW = negated ? -rho : rho; + const VectorType dW = negated ? VectorType(-d) : d; // pi maps each sorted working index to its input row. std::vector<Index> pi; @@ -340,61 +179,27 @@ zs[i] = z[pi[static_cast<std::size_t>(i)]]; } - // Normalize z and absorb ||z||^2 into rho. Power-of-two scaling is exact away - // from the subnormal boundary, unlike the general scaling in LAPACK xLAED*. + // Normalize z and absorb ||z||^2 into rho, then scale the problem by the exact + // power of two s = 2^-scaleExp that brings max(||D||_inf, rho ||z||^2) into + // [1/2, 1): eig(sD + s rho zz^T) = s eig(D + rho zz^T). + const RealScalar znorm = zs.stableNorm(); + if (znorm > RealScalar(0)) zs /= znorm; + RealScalar rhoW = rho == RealScalar(0) || znorm == RealScalar(0) ? RealScalar(0) : (numext::abs(rho) * znorm) * znorm; + if (!(numext::isfinite)(rhoW)) { + // An overflowing update would make the deflation tolerance infinite and + // silently deflate it away. + m_eivalues.setConstant(NumTraits<RealScalar>::quiet_NaN()); + m_info = InvalidInput; + m_isInitialized = true; + return *this; + } EIGEN_USING_STD(frexp) EIGEN_USING_STD(ldexp) - // With max|z_i| < 2^zExp, ||2^-zExp z|| is representable even if ||z|| is not. - int zExp = 0; - const RealScalar zmax = zs.cwiseAbs().maxCoeff(); - if (zmax > RealScalar(0)) { - frexp(zmax, &zExp); - if (zExp != 0) zs.array() = zs.array().ldexp(-zExp); - } - const RealScalar znorm = zs.stableNorm(); // in [0.5, sqrt(n)): safe - int znormExp = 0; - const RealScalar znormFrac = frexp(znorm, &znormExp); - if (znorm > RealScalar(0)) zs /= znorm; - // Store rho ||z||^2 = rhoMant * 2^rhoTotExp without materializing a possibly - // overflowing product; each mantissa factor lies in [1/4,1). - int rhoExp = 0; - const RealScalar rhoFrac = frexp(rhoW, &rhoExp); - int rhoAdj = 0; - const RealScalar rhoMant = frexp((rhoFrac * znormFrac) * znormFrac, &rhoAdj); // in [0.5, 1), or 0 - // Each frexp exponent is bounded by the scalar's exponent range, but their - // accumulation is kept in a wide integer type until the (clamped) narrowing - // to the int that ldexp takes. - const numext::int64_t rhoTotExp = - numext::int64_t(rhoExp) + 2 * (numext::int64_t(zExp) + numext::int64_t(znormExp)) + numext::int64_t(rhoAdj); - // For s = 2^-scaleExp, eig(sD + s rho zz^T) = s eig(D + rho zz^T). - // Choose s so max(||sD||_inf, s rho ||z||^2) lies in [1/2,1), comparing the - // two scales as mantissa-exponent pairs to avoid overflow. - int dExp = 0; - const RealScalar dFrac = frexp(ds.cwiseAbs().maxCoeff(), &dExp); - RealScalar scaledNorm; // max(|d|_inf, rho*||z||^2) * 2^-scaleExp, in [0.5, 1) (or 0 for a zero matrix) - numext::int64_t scaleExpWide; - if (rhoMant == RealScalar(0) || - (dFrac > RealScalar(0) && - (numext::int64_t(dExp) > rhoTotExp || (numext::int64_t(dExp) == rhoTotExp && dFrac >= rhoMant)))) { - scaledNorm = dFrac; - scaleExpWide = dExp; - } else { - scaledNorm = rhoMant; - scaleExpWide = rhoTotExp; - } - // Clamp before narrowing: 2^(2^30) is far beyond any scalar's exponent - // range, so the clamp never changes which values are representable. - const numext::int64_t expCap = numext::int64_t(1) << 30; - const int scaleExp = static_cast<int>(numext::maxi(-expCap, numext::mini(expCap, scaleExpWide))); - SpectrumRange spectrumRange = SpectrumRange::Representable; - if (scaleExpWide >= numext::int64_t(std::numeric_limits<RealScalar>::max_exponent) - 1) { - spectrumRange = classifySpectrumRange(dW, rhoW, z); - } - if (scaleExp != 0) ds.array() = ds.array().ldexp(-scaleExp); - // Materialize rho * ||z||^2 only in scaled form: its exponent is <= 0 by the - // choice of scaleExp, so this cannot overflow; it can only underflow when - // the update is negligible against |d|_inf, in which case it deflates below. - rhoW = ldexp(rhoMant, static_cast<int>(numext::maxi(-expCap, rhoTotExp - scaleExpWide))); + RealScalar scaledNorm = numext::maxi(ds.cwiseAbs().maxCoeff(), rhoW); // in [1/2, 1) once scaled + int scaleExp = 0; + if (scaledNorm > RealScalar(0)) scaledNorm = frexp(scaledNorm, &scaleExp); + ds = ds.array().ldexp(-scaleExp).matrix(); + rhoW = ldexp(rhoW, -scaleExp); // Backward-error budget: dropping a coupling of size <= tol perturbs the // matrix by O(tol), like LAPACK's xLAED2. Using max(|d|_inf, rho*||z||^2) as @@ -419,17 +224,30 @@ if (p >= 0) { const RealScalar r = numext::hypot(zs[p], zs[i]); const RealScalar c = zs[i] / r, s = zs[p] / r; // zeroes the earlier entry - if (numext::abs(c * s * (ds[i] - ds[p])) <= tol) { - // Rotated diagonal: the deflated position keeps one entry, the - // surviving pole the other; both stay inside [ds[p], ds[i]]. - const RealScalar dp = c * c * ds[p] + s * s * ds[i]; - const RealScalar di = s * s * ds[p] + c * c * ds[i]; - ds[p] = dp; - ds[i] = di; + const RealScalar gap = ds[i] - ds[p]; + if (numext::abs(c * s * gap) <= tol) { + // Rotated diagonal (c^2 a + s^2 b, s^2 a + c^2 b), a = ds[p], b = ds[i], with c^2 + s^2 = 1 + // applied exactly and the smaller weight w = min(c^2, s^2) <= 1/2 on the gap: + // |s| <= |c|: (a + s^2 gap, b - s^2 gap), otherwise (b - c^2 gap, a + c^2 gap). + // Equal poles (gap = 0) keep their value, which the weighted sums lose to the rounding of + // c^2 + s^2, and the rounding of a wide gap enters scaled by w. + const RealScalar left = ds[p], right = ds[i]; + if (numext::abs(s) <= numext::abs(c)) { + const RealScalar shift = s * s * gap; + ds[p] = left + shift; + ds[i] = right - shift; + } else { + const RealScalar shift = c * c * gap; + ds[p] = right - shift; + ds[i] = left + shift; + } zs[i] = r; zs[p] = RealScalar(0); deflated[static_cast<std::size_t>(p)] = true; - if (computeVectors) rotations.push_back(Rotation{p, i, c, s}); + // Recorded for both options, keeping computeVectors out of the loops that produce the + // eigenvalues: GCC unswitches this loop on it, and IBM double-double operations are not + // commutative, so the two copies' operand orders can round differently. + rotations.push_back(Rotation{p, i, c, s}); } } if (!deflated[static_cast<std::size_t>(i)]) p = i; @@ -587,29 +405,10 @@ for (Index i = 0; i < n; ++i) m_eivec(pi[static_cast<std::size_t>(i)], outCol) = wvec[i]; } - // Undo the problem scaling in one full-range pass. ArrayBase::ldexp handles - // exponents for which 2^scaleExp is not itself representable, preserving - // representable results exactly and saturating genuine overflow to infinity - // (validated below). - if (scaleExp != 0) m_eivalues.array() = m_eivalues.array().ldexp(scaleExp); - - // Range classification uses the original data. The normalized secular - // problem can round a root across the maximum-finite boundary before this - // rescaling, so finiteness of the rounded result alone is not evidence that - // the exact spectrum is (or is not) representable. - const Index extreme = negated ? 0 : n - 1; - const RealScalar highest = (std::numeric_limits<RealScalar>::max)(); - if (spectrumRange == SpectrumRange::Overflow) { + m_eivalues = m_eivalues.array().ldexp(scaleExp).matrix(); + if (!m_eivalues.allFinite()) { m_eivalues.setConstant(NumTraits<RealScalar>::quiet_NaN()); m_info = InvalidInput; - } else { - // A proven boundary root and a root the rescaling saturated to infinity both belong at the - // largest finite value: the first is exactly it, the second rounded across it. - if (spectrumRange == SpectrumRange::ExactBoundary || !(numext::isfinite)(m_eivalues[extreme])) { - m_eivalues[extreme] = negated ? -highest : highest; - } - if (spectrumRange == SpectrumRange::Uncertain && m_info == Success) m_info = NoConvergence; - if (!m_eivalues.allFinite() && m_info == Success) m_info = NoConvergence; } m_vectorsComputed = computeVectors && m_info != InvalidInput;
diff --git a/contrib/Eigen/src/StructuredMatrices/DiagonalPlusLowRank.h b/contrib/Eigen/src/StructuredMatrices/DiagonalPlusLowRank.h index 18df0a0..1d7df6e 100644 --- a/contrib/Eigen/src/StructuredMatrices/DiagonalPlusLowRank.h +++ b/contrib/Eigen/src/StructuredMatrices/DiagonalPlusLowRank.h
@@ -20,11 +20,8 @@ // (fraction, exponent) pair to avoid spurious overflow/underflow. // [4] P. H. Sterbenz, "Floating-Point Computation", Prentice-Hall, 1974. // Scaling by a power of two is exact, the property the balanced -// accumulation and the normalized Woodbury products rely on. -// [5] N. J. Higham, "Accuracy and Stability of Numerical Algorithms", 2nd ed., -// SIAM, 2002, chapter 27. Overflow-avoiding rescaling of intermediate -// quantities, the technique behind the normalized capacitance products. -// [6] E. L. Yip, "A Note on the Stability of Solving a Rank-p Modification of +// accumulation relies on. +// [5] E. L. Yip, "A Note on the Stability of Solving a Rank-p Modification of // a Linear System by the Sherman-Morrison-Woodbury Formula", SIAM Journal // on Scientific and Statistical Computing, 7(3), pp. 507-513, 1986. The // accuracy limitation of the Woodbury solve noted on solve(). @@ -65,77 +62,25 @@ using Shape = StructuredShape; }; -// Entrywise multiplication by the exact power of two 2^e [4], for the factor normalization of the -// Woodbury products [5]. Uses structured_balance_impl::apply_exponent (an ldexp per component) rather -// than a multiplication by 2^e or by two half powers 2^(e/2): ldexp is exact and saturates entrywise -// for every exponent, whereas 2^-eu itself overflows for a subnormal factor bound and a saturated -// half power turns zero entries into NaN through 0 * Inf. -template <typename Scalar> -struct dplr_ldexp_op { - int e; - Scalar operator()(const Scalar& x) const { return structured_balance_impl<Scalar>::apply_exponent(x, e); } -}; - -// The smallest e with n <= 2^e: the inner-dimension term of a product's entry -// magnitude bound (a length-n dot product of entries below 2^a and 2^b stays -// below 2^(a + b + e)). -inline int dplr_index_exponent(Index n) { return log2_ceil(static_cast<std::make_unsigned_t<Index>>(n)); } - -// True when a product of operands with entry-magnitude exponent bounds ea and eb over an inner -// dimension of length \a inner provably cannot overflow, with one bit of headroom for a trailing -// addition. Where it fails the Woodbury kernels rescale their factors exactly [4][5]; where it holds -// they keep the plain association. -template <typename RealScalar> -bool dplr_product_fits(int ea, int eb, Index inner) { - return ea + eb + dplr_index_exponent(inner) + 1 < NumTraits<RealScalar>::max_exponent(); -} - // Compile-time dispatch keeps fixed rank zero from instantiating a 0 x 0 LU; // C++14 cannot express this branch with if constexpr. template <int Rank_> struct dplr_capacitance_impl { /** Applies the Woodbury correction - * \f[ x\mathrel{-}=D^{-1}U(I_k+V^H D^{-1}U)^{-1}V^Hx. \f] - * Exact power-of-two scaling is used only when either structural product can - * overflow [4][5], preserving the unscaled result bit-for-bit otherwise. */ + * \f[ x\mathrel{-}=D^{-1}U(I_k+V^H D^{-1}U)^{-1}V^Hx. \f] */ template <typename Op, typename Dinv, typename Workspace> static void subtractSolveCorrection(const Op& op, const Dinv& dinv, Workspace& x) { - using Scalar = typename Op::Scalar; - using RealScalar = typename Op::RealScalar; if (op.correctionRank() == 0 || x.size() == 0) return; PartialPivLU<typename Op::CapacitanceType> cap(op.capacitance()); - const int eu = structured_exponent_bound(op.factorU()); - const int ev = structured_exponent_bound(op.factorV()); - const int ed = structured_exponent_bound(dinv); - const int ex = structured_exponent_bound(x); - if (dplr_product_fits<RealScalar>(ev, ex, op.rows()) && dplr_product_fits<RealScalar>(ed, eu, 1)) { - x.noalias() -= dinv.asDiagonal() * (op.factorU() * cap.solve(op.factorV().adjoint() * x)); - return; - } - Matrix<Scalar, Rank_, Workspace::ColsAtCompileTime> y = - cap.solve(op.factorV().unaryExpr(dplr_ldexp_op<Scalar>{-ev}).adjoint() * x); - y = y.unaryExpr(dplr_ldexp_op<Scalar>{eu + ev}); - x.noalias() -= dinv.asDiagonal() * (op.factorU().unaryExpr(dplr_ldexp_op<Scalar>{-eu}) * y); + x.noalias() -= dinv.asDiagonal() * (op.factorU() * cap.solve(op.factorV().adjoint() * x)); } - /** Forms \f$U'=-D^{-1}U(I_k+V^HD^{-1}U)^{-1}\f$. Scaling prevents a - * spurious overflow in \f$D^{-1}U\f$ when the capacitance inverse subsequently - * shrinks it [4][5]. */ + /** Forms \f$U'=-D^{-1}U(I_k+V^HD^{-1}U)^{-1}\f$. */ template <typename Op, typename Dinv, typename Factor> static void inverseFactor(const Op& op, const Dinv& dinv, Factor& Up) { - using Scalar = typename Op::Scalar; - using RealScalar = typename Op::RealScalar; if (op.correctionRank() == 0) return; PartialPivLU<typename Op::CapacitanceType> cap(op.capacitance()); const typename Op::CapacitanceType K = cap.inverse(); - const int eu = structured_exponent_bound(op.factorU()); - const int ed = structured_exponent_bound(dinv); - const int ek = structured_exponent_bound(K); - if (dplr_product_fits<RealScalar>(ed, eu, 1) && dplr_product_fits<RealScalar>(ed + eu, ek, op.correctionRank())) { - Up.noalias() = -(dinv.asDiagonal() * op.factorU() * K); - return; - } - Up.noalias() = -(dinv.asDiagonal() * op.factorU().unaryExpr(dplr_ldexp_op<Scalar>{-eu}) * K); - Up = Up.unaryExpr(dplr_ldexp_op<Scalar>{eu}); + Up.noalias() = -(dinv.asDiagonal() * op.factorU() * K); } /** Accumulates the determinant-lemma factor * \f$\det(I_k+V^HD^{-1}U)\f$ as a mantissa and power of two. Each LU pivot is @@ -268,38 +213,16 @@ /** \returns the capacitance matrix \f$ I_k + V^H D^{-1} U \f$ of the Woodbury * identity. Every consumer of the triple product -- \ref solve, \ref inverse - * and \ref determinant -- forms it through this one method. - * - * With extreme factor magnitudes the plain association can overflow even - * though the capacitance itself is representable: for \c d = 1e-200, - * \c U = 1e-200, \c V = 1e200 the operator is essentially the identity and - * the capacitance \c 1 + 1e200 is representable, but \f$ V^H D^{-1} \f$ is - * \c 1e400. The factors are therefore rescaled by exact powers of two [4] - * when a conservative exponent bound detects the danger, the product is - * formed as \f$ \hat V^H (D^{-1} \hat U) \f$ -- whose intermediates are - * bounded by roughly \c n * max|1/d| -- and the removed exponent is folded - * back entrywise before the identity is added [5]. When the plain - * association provably cannot overflow it is kept, so results for moderate - * data are bit-identical to the unnormalized evaluation. + * and \ref determinant -- forms it through this one method. With extreme + * factor magnitudes the product can overflow even when the capacitance itself + * is representable. * \warning The diagonal must have no zero entries. */ CapacitanceType capacitance() const { const Index k = correctionRank(); CapacitanceType c = CapacitanceType::Identity(k, k); if (k == 0) return c; const DiagonalVector dinv = m_d.cwiseInverse(); - const int eu = internal::structured_exponent_bound(m_U); - const int ev = internal::structured_exponent_bound(m_V); - const int ed = internal::structured_exponent_bound(dinv); - if (internal::dplr_product_fits<RealScalar>(ev, ed, 1) && - internal::dplr_product_fits<RealScalar>(ev + ed, eu, rows())) { - c.noalias() += m_V.adjoint() * dinv.asDiagonal() * m_U; - return c; - } - const FactorType W = dinv.asDiagonal() * m_U.unaryExpr(internal::dplr_ldexp_op<Scalar>{-eu}); - CapacitanceType t(k, k); - t.noalias() = m_V.unaryExpr(internal::dplr_ldexp_op<Scalar>{-ev}).adjoint() * W; - t = t.unaryExpr(internal::dplr_ldexp_op<Scalar>{eu + ev}); - c += t; + c.noalias() += m_V.adjoint() * dinv.asDiagonal() * m_U; return c; } @@ -312,9 +235,7 @@ * \warning The Woodbury splitting routes the solution through \f$ D^{-1} b \f$: * when \c max|1/d| greatly exceeds the norm of the operator's inverse, the * correction cancels most of those amplified digits and the achievable - * accuracy degrades accordingly, even for a well-conditioned operator [6]. - * The normalized kernels keep such solves finite; they cannot restore the - * cancelled digits. */ + * accuracy degrades accordingly, even for a well-conditioned operator [5]. */ template <typename Rhs> Matrix<Scalar, Size_, Rhs::ColsAtCompileTime> solve(const MatrixBase<Rhs>& b) const { EIGEN_STATIC_ASSERT(RowsAtCompileTime == Dynamic || Rhs::RowsAtCompileTime == Dynamic || @@ -322,20 +243,8 @@ YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES) eigen_assert(b.rows() == rows() && "right-hand side has the wrong number of rows"); const DiagonalVector dinv = m_d.cwiseInverse(); - // D^{-1} b can overflow while the solution is representable: at d = 1e-200, b = 1e200 the operator - // is essentially the identity and x is 1e200, yet this term is 1e400. The solve is linear in b, so - // b is rescaled by an exact power of two [4] on the same exponent test the other kernels use and - // the exponent is folded back into the result [5]; the plain form is kept where it provably - // cannot overflow. - const int eb = internal::structured_exponent_bound(b.derived()); - const int ed = internal::structured_exponent_bound(dinv); - const bool rescale = !internal::dplr_product_fits<RealScalar>(ed, eb, 1); - - Matrix<Scalar, Size_, Rhs::ColsAtCompileTime> x = - rescale ? (dinv.asDiagonal() * b.derived().unaryExpr(internal::dplr_ldexp_op<Scalar>{-eb})).eval() - : (dinv.asDiagonal() * b).eval(); + Matrix<Scalar, Size_, Rhs::ColsAtCompileTime> x = dinv.asDiagonal() * b; internal::dplr_capacitance_impl<Rank_>::subtractSolveCorrection(*this, dinv, x); - if (rescale) x = x.unaryExpr(internal::dplr_ldexp_op<Scalar>{eb}); return x; } @@ -356,13 +265,16 @@ * Both factors are accumulated in the balanced form \c m * 2^e (the split * fraction/exponent determinant convention of LINPACK's xGEDI [3]) -- the * diagonal entries and the LU pivots of the capacitance matrix are - * renormalized to unit magnitude one at a time, with the powers of two tracked + * renormalized to unit magnitude one at a time [4], with the powers of two tracked * in a shared exponent - * -- so no partial product, in particular neither ordinary determinant on its - * own, can overflow or underflow when the combined determinant is - * representable, whatever the ordering and magnitudes of the entries. - * Genuinely out-of-range determinants still saturate to (signed) zero or - * infinity. + * -- so neither ordinary determinant on its own can overflow or underflow + * when the combined determinant is representable, whatever the ordering and + * magnitudes of the entries. Genuinely out-of-range determinants still + * saturate to (signed) zero or infinity. The capacitance entries themselves + * are formed in plain arithmetic (see \ref capacitance): at extreme factor + * magnitudes such as \c d = 1e-200, \c U = 1e-200, \c V = 1e200 they, and + * with them the determinant, overflow although the determinant (~1) is + * representable. * \warning The diagonal must have no zero entries (use the lemma symmetrically * or a dense fallback for that case), and, as with \ref solve and \ref inverse, * its reciprocals must be finite: a subnormal diagonal entry overflows
diff --git a/contrib/Eigen/src/StructuredMatrices/Hankel.h b/contrib/Eigen/src/StructuredMatrices/Hankel.h index 22b9aaa..7bbf12e 100644 --- a/contrib/Eigen/src/StructuredMatrices/Hankel.h +++ b/contrib/Eigen/src/StructuredMatrices/Hankel.h
@@ -140,7 +140,6 @@ if (n > 1) m_h.tail(n - 1) = row.tail(n - 1); if (m > 1 && n > 1 && (m > internal::structured_direct_threshold() || n > internal::structured_direct_threshold())) m_symbol = computeSymbol(); - m_fftUsable = computeFftUsable(); } EIGEN_DEVICE_FUNC Index rows() const { return m_rows.value(); } @@ -275,28 +274,18 @@ * The FFT path evaluates the circular convolution * \f[ (Hx)_i = \sum_{j=0}^{n-1} h_{i+j} x_j. \f] * Reversing \f$x\f$ and rotating the padded symbol by \f$n-1\f$ places these - * entries at the start of the inverse transform. - * - * Non-finite data takes the direct kernels: the transforms would smear a - * single Inf/NaN into NaNs across the whole output, where the dense product - * only propagates it through the dot products that touch it. A non-finite - * generating sequence or cached symbol (which can overflow even for a finite - * sequence) routes the whole product; a non-finite right-hand-side column is - * detected inside the FFT loop -- in the same pass that derives its scaling - * exponent, so finite data pays no extra scan -- and falls back per column. */ + * entries at the start of the inverse transform. */ template <typename Dest, typename Rhs, typename ProductScalar> void addProduct(Dest& dst, const Rhs& rhs, const ProductScalar& alpha) const { const Index m = rows(), n = cols(); eigen_assert(rhs.rows() == n && "invalid product: dimensions do not match"); const bool small = m == 1 || n == 1 || (m <= internal::structured_direct_threshold() && n <= internal::structured_direct_threshold()); - if (small || !m_fftUsable) { + if (small) { directProduct(dst, rhs, alpha); return; } - // Reversal preserves which right-hand-side columns need the direct fallback. - internal::structured_fft_apply(dst, m_symbol, m, rhs.colwise().reverse(), alpha, - [&](Index k) { directProductColumn(dst, rhs, k, alpha); }); + internal::structured_fft_apply(dst, m_symbol, m, rhs.colwise().reverse(), alpha); } private: @@ -308,20 +297,12 @@ * public constructor. Used by transpose(), conjugate() and adjoint(), whose * symbols are cheap transformations of the existing one. */ Hankel(const GeneratorType& h, Index rows, Index cols, const ComplexVector& symbol) - : m_rows(rows), m_cols(cols), m_h(h), m_symbol(symbol), m_fftUsable(computeFftUsable()) {} - - /** \internal Whether products may take the FFT path: the generating sequence - * and the cached symbol must be finite. The symbol accumulates up to p addends, - * so it can overflow to Inf even for a finite sequence; such operators fall - * back to the direct kernels, which stay exact. */ - bool computeFftUsable() const { return m_h.allFinite() && (m_symbol.size() == 0 || m_symbol.allFinite()); } + : m_rows(rows), m_cols(cols), m_h(h), m_symbol(symbol) {} /** \internal Direct kernel for column \a k of the right-hand side: computes * \c dst.col(k) += alpha * (*this) * rhs.col(k) without transforms. Serves - * operators below the FFT threshold, single-row and single-column operators - * (whose products cost O(n) directly, however large the operator), and any - * column involving non-finite data, whose entrywise IEEE semantics the - * transforms cannot preserve. */ + * operators below the FFT threshold and single-row and single-column operators + * (whose products cost O(n) directly, however large the operator). */ template <typename Dest, typename Rhs, typename ProductScalar> void directProductColumn(Dest& dst, const Rhs& rhs, Index k, const ProductScalar& alpha) const { const Index m = rows(), n = cols(); @@ -424,7 +405,6 @@ internal::variable_if_dynamic<Index, ColsAtCompileTime> m_cols; GeneratorType m_h; ComplexVector m_symbol; - bool m_fftUsable; }; /** \ingroup StructuredMatrices_Module
diff --git a/contrib/Eigen/src/StructuredMatrices/KroneckerOperator.h b/contrib/Eigen/src/StructuredMatrices/KroneckerOperator.h index 964e280..12d72d1 100644 --- a/contrib/Eigen/src/StructuredMatrices/KroneckerOperator.h +++ b/contrib/Eigen/src/StructuredMatrices/KroneckerOperator.h
@@ -10,19 +10,18 @@ // References: // [1] C. F. Van Loan, "The ubiquitous Kronecker product", Journal of // Computational and Applied Mathematics 123 (2000), 85-100. The vec -// identity driving every product and solve below -- stated there as +// identity, with vec stacking columns, driving every product and solve +// below -- stated there as // Y = C X B^T <=> vec(Y) = (B (x) C) vec(X), i.e. with this file's // operand naming (A (x) B) vec(X) = vec(B X A^T) -- together with the // factor-wise identities for the inverse, pseudo-inverse, // eigendecomposition, SVD and determinant of a Kronecker product. // [2] N. J. Higham, "Accuracy and Stability of Numerical Algorithms", 2nd ed., // SIAM, 2002, chapter 27. Avoiding spurious overflow by rescaling with -// powers of two, the technique behind the scaled vec-trick products, the -// normalized-frame solves, the exponent-split pseudo-inversion and the -// exponent-balanced determinant. +// powers of two, the technique behind the exponent-balanced determinant. // [3] P. H. Sterbenz, "Floating-Point Computation", Prentice-Hall, 1974. -// Scaling by a power of two is exact, the property every rescaling in this -// file relies on. +// Scaling by a power of two is exact, the property the determinant's +// rescaling relies on. #ifndef EIGEN_STRUCTURED_KRONECKER_OPERATOR_H #define EIGEN_STRUCTURED_KRONECKER_OPERATOR_H @@ -66,11 +65,41 @@ using Shape = StructuredShape; }; -// Three factor kinds -- dense, diagonal, sparse -- with every kind-specific -// operation in the kron_factor_* helpers below, so the operator has a single -// implementation. Diagonal: stored as its diagonal, applied as a scaling, solved -// entrywise, closed under transposition, inverse and determinant. Sparse: stored -// compressed, applied by sparse-dense products, solved via SparseLU; inverse dense. +// Factor kinds, each with its operations in the kron_factor_* helpers below: +// dense; diagonal (stored as its diagonal); sparse (compressed, solved by +// SparseLU); identity (a kron_identity_factor, dimensions only, skipped in +// products); Kronecker (a nested KroneckerOperator, for three or more factors). + +template <typename Scalar_, int Rows_, int Cols_> +class kron_identity_factor; + +template <typename Scalar_, int Rows_, int Cols_> +struct traits<kron_identity_factor<Scalar_, Rows_, Cols_>> : traits<Matrix<Scalar_, Rows_, Cols_>> {}; + +/** \internal The identity factor: the m x n matrix with ones on the main + * diagonal, stored as its dimensions. makeKroneckerOperator() stores an + * Identity() expression as one, since the expression is not assignable and + * would make every operator holding it unassignable too. */ +template <typename Scalar_, int Rows_, int Cols_> +class kron_identity_factor : public EigenBase<kron_identity_factor<Scalar_, Rows_, Cols_>> { + public: + using Scalar = Scalar_; + using PlainObject = Matrix<Scalar, Rows_, Cols_>; + static constexpr int RowsAtCompileTime = Rows_; + static constexpr int ColsAtCompileTime = Cols_; + + kron_identity_factor(Index rows, Index cols) : m_rows(rows), m_cols(cols) {} + template <typename PlainObjectType> + explicit kron_identity_factor(const CwiseNullaryOp<scalar_identity_op<Scalar>, PlainObjectType>& identity) + : m_rows(identity.rows()), m_cols(identity.cols()) {} + + EIGEN_DEVICE_FUNC constexpr Index rows() const { return m_rows.value(); } + EIGEN_DEVICE_FUNC constexpr Index cols() const { return m_cols.value(); } + + private: + variable_if_dynamic<Index, Rows_> m_rows; + variable_if_dynamic<Index, Cols_> m_cols; +}; template <typename Factor> struct kron_factor_is_diagonal : std::false_type {}; @@ -87,37 +116,49 @@ template <typename Scalar, int Options, typename StorageIndex> struct kron_factor_is_sparse_matrix<SparseMatrix<Scalar, Options, StorageIndex>> : std::true_type {}; +template <typename Factor> +struct kron_factor_is_identity : std::false_type {}; +template <typename Scalar, int Rows, int Cols> +struct kron_factor_is_identity<kron_identity_factor<Scalar, Rows, Cols>> : std::true_type {}; + +template <typename Factor> +struct kron_factor_is_kronecker : std::false_type {}; +template <typename LhsMatrix, typename RhsMatrix> +struct kron_factor_is_kronecker<KroneckerOperator<LhsMatrix, RhsMatrix>> : std::true_type {}; + // The factor kind, the dispatch key of kron_factor_ops and kron_factor_solver. constexpr int kKronDenseFactor = 0; constexpr int kKronDiagonalFactor = 1; constexpr int kKronSparseFactor = 2; +constexpr int kKronIdentityFactor = 3; +constexpr int kKronKroneckerFactor = 4; template <typename Factor> constexpr int kron_factor_kind() { return kron_factor_is_diagonal<Factor>::value ? kKronDiagonalFactor : kron_factor_is_sparse_matrix<Factor>::value ? kKronSparseFactor + : kron_factor_is_identity<Factor>::value ? kKronIdentityFactor + : kron_factor_is_kronecker<Factor>::value ? kKronKroneckerFactor : kKronDenseFactor; } -// Apply 2^e without forming a possibly unrepresentable scale factor. For -// std::complex storage, realView exposes both components to the real ldexp packets. -template <typename Xpr, std::enable_if_t<!NumTraits<typename Xpr::Scalar>::IsComplex, bool> = true> -void kron_ldexp_entries(Xpr& M, int e) { - M.array() = M.array().ldexp(e); -} - -template <typename Xpr, std::enable_if_t<complex_array_access<typename Xpr::Scalar>::value, bool> = true> -void kron_ldexp_entries(Xpr& M, int e) { - M.realView().array() = M.realView().array().ldexp(e); -} - -template <typename Xpr, std::enable_if_t<NumTraits<typename Xpr::Scalar>::IsComplex && - !complex_array_access<typename Xpr::Scalar>::value, - bool> = true> -void kron_ldexp_entries(Xpr& M, int e) { - using Scalar = typename Xpr::Scalar; - M = M.unaryExpr([e](const Scalar& z) { return structured_ldexp_clamped(z, Index(e)); }); -} +/** \internal The type makeKroneckerOperator() stores an argument as: its plain + * object, except an Identity() expression, stored as its kron_identity_factor, + * and an identity factor or a KroneckerOperator, which own everything they need + * already. */ +template <typename Derived, + bool StoredAsIs = kron_factor_is_identity<Derived>::value || kron_factor_is_kronecker<Derived>::value> +struct kron_factor_storage { + using type = typename Derived::PlainObject; +}; +template <typename Derived> +struct kron_factor_storage<Derived, true> { + using type = Derived; +}; +template <typename Scalar, typename PlainObjectType> +struct kron_factor_storage<CwiseNullaryOp<scalar_identity_op<Scalar>, PlainObjectType>, false> { + using type = kron_identity_factor<Scalar, PlainObjectType::RowsAtCompileTime, PlainObjectType::ColsAtCompileTime>; +}; template <typename Factor, int Kind = kron_factor_kind<Factor>()> struct kron_factor_ops { @@ -151,22 +192,35 @@ static auto inversed(const Factor& f) { return f.inverse(); } // The factor as a dense expression, for the decomposition family. static const Factor& denseFactor(const Factor& f) { return f; } - // The right operand op(F) of the vec-trick product Y = B X op(A): the - // transpose for a dense factor, the factor itself for a diagonal one. - static auto transposedOperand(const Factor& f) { return f.transpose(); } - static bool allFinite(const Factor& f) { return f.allFinite(); } + // The factor as the operand of a dense block assignment dst = a * f. + static const Factor& blockOperand(const Factor& f) { return f; } + static bool isSquareIdentity(const Factor&) { return false; } + // dst += alpha F X and dst += alpha X F^T: the two sides of the vec-trick product B X A^T. + // work: scratch for a nested factor's right product, owned by the caller's loop. + template <typename Dst, typename Alpha, typename Xpr> + static void addLeftProduct(Dst& dst, const Alpha& alpha, const Factor& f, const Xpr& X) { + dst.noalias() += alpha * (f * X); + } + template <typename Dst, typename Alpha, typename Xpr, typename Work> + static void addRightProduct(Dst& dst, const Alpha& alpha, const Xpr& X, const Factor& f, Work&) { + dst.noalias() += alpha * (X * f.transpose()); + } static int exponentBound(const Factor& f) { return structured_exponent_bound(f); } - /** \internal \returns the dot-product growth bits of a product contracting - * over \a n entries of this factor. Cast to an unsigned type deliberately: - * the generic count_bits_impl fallback static_asserts on unsignedness, and - * only the GCC/Clang and MSVC specialisations accept a signed Index. */ - static int growthBits(Index n) { return log2_floor(static_cast<numext::uint64_t>(n)) + 1; } /** \internal \returns the mantissa of \c det(M) in the balanced form * \c m * 2^e, adding \c e into \a exponent: the determinant is accumulated * directly from the LU diagonal (times the permutation sign), each entry and * the running product renormalized by \c structured_balance. */ static Scalar balancedDet(const Factor& M, Index& exponent) { - PartialPivLU<Factor> lu(M); + // Scale small factors up, exactly, so that the elimination runs on normal + // numbers: a subnormal near 2^e carries only e - min_exponent + digits bits. + // Scaling down could erase small pivots in factors with a wide exponent range, + // and every entry is scaled exactly because a small entry can be a pivot. + const int bound = exponentBound(M); + const int scaleExponent = numext::mini(bound, 0); + Factor normalized = M; + structured_ldexp_entries_exact(normalized, -scaleExponent); + exponent += M.rows() * scaleExponent; + PartialPivLU<Factor> lu(normalized); Scalar m = Scalar(RealScalar(lu.permutationP().determinant())); // +-1 for (Index i = 0; i < M.rows(); ++i) m = structured_balance(m * structured_balance(lu.matrixLU().coeff(i, i), exponent), exponent); @@ -200,15 +254,27 @@ static auto adjointed(const Factor& f) { return f.diagonal().conjugate().asDiagonal(); } static auto inversed(const Factor& f) { return f.inverse(); } static typename Factor::DenseMatrixType denseFactor(const Factor& f) { return f.toDenseMatrix(); } - static const Factor& transposedOperand(const Factor& f) { return f; } - static bool allFinite(const Factor& f) { return f.diagonal().allFinite(); } + static const Factor& blockOperand(const Factor& f) { return f; } + static bool isSquareIdentity(const Factor&) { return false; } + template <typename Dst, typename Alpha, typename Xpr> + static void addLeftProduct(Dst& dst, const Alpha& alpha, const Factor& f, const Xpr& X) { + dst.noalias() += alpha * (f * X); + } + template <typename Dst, typename Alpha, typename Xpr, typename Work> + static void addRightProduct(Dst& dst, const Alpha& alpha, const Xpr& X, const Factor& f, Work&) { + dst.noalias() += alpha * (X * f); + } static int exponentBound(const Factor& f) { return structured_exponent_bound(f.diagonal()); } - // A diagonal product has no accumulation, hence no dot-product growth. - static int growthBits(Index) { return 0; } static Scalar balancedDet(const Factor& D, Index& exponent) { + // Scale a small diagonal up first: the balancing below compares and scales + // in floating point, which reads a subnormal entry as zero under flush-to-zero. + const int bound = exponentBound(D); + const int scaleExponent = numext::mini(bound, 0); + typename Factor::DiagonalVectorType d = D.diagonal(); + structured_ldexp_entries_exact(d, -scaleExponent); + exponent += D.rows() * scaleExponent; Scalar m(1); - for (Index i = 0; i < D.rows(); ++i) - m = structured_balance(m * structured_balance(D.diagonal().coeff(i), exponent), exponent); + for (Index i = 0; i < D.rows(); ++i) m = structured_balance(m * structured_balance(d.coeff(i), exponent), exponent); return m; } }; @@ -289,20 +355,27 @@ return InverseFactor::Constant(rows, cols, Scalar(NumTraits<RealScalar>::quiet_NaN())); } static InverseFactor denseFactor(const Factor& f) { return InverseFactor(f); } - static auto transposedOperand(const Factor& f) { return f.transpose(); } - static bool allFinite(const Factor& f) { return f.coeffs().allFinite(); } + static const Factor& blockOperand(const Factor& f) { return f; } + static bool isSquareIdentity(const Factor&) { return false; } + template <typename Dst, typename Alpha, typename Xpr> + static void addLeftProduct(Dst& dst, const Alpha& alpha, const Factor& f, const Xpr& X) { + dst.noalias() += alpha * (f * X); + } + template <typename Dst, typename Alpha, typename Xpr, typename Work> + static void addRightProduct(Dst& dst, const Alpha& alpha, const Xpr& X, const Factor& f, Work&) { + dst.noalias() += alpha * (X * f.transpose()); + } static int exponentBound(const Factor& f) { return structured_exponent_bound(f.coeffs()); } - // The dense contraction bound: a row holds at most n stored entries. - static int growthBits(Index n) { return log2_floor(static_cast<numext::uint64_t>(n)) + 1; } static Scalar balancedDet(const Factor& M, Index& exponent) { // Scale small factors up, exactly, so that the elimination runs on normal // numbers: a subnormal near 2^e carries only e - min_exponent + digits bits. // Scaling down could erase small pivots in factors with a wide exponent range. - const int scaleExponent = numext::mini(exponentBound(M), 0); + const int bound = exponentBound(M); + const int scaleExponent = numext::mini(bound, 0); ColMajorFactor normalized(M); if (scaleExponent != 0) { auto values = normalized.coeffs(); - kron_ldexp_entries(values, -scaleExponent); + structured_ldexp_entries_exact(values, -scaleExponent); } kron_sparse_lu<ColMajorFactor> lu; lu.compute(normalized); @@ -316,40 +389,153 @@ } }; -/** \internal Per-factor solve adapter for KroneckerOperator::solve(). All - * kinds normalize the factor by an exact power of two up front and then solve - * per right-hand-side column in the shared normalized frame; the caller folds - * the removed \c exponent() back together with the other factor's and the - * column's. A dense factor is LU-factorized once, a sparse factor once by - * SparseLU; a diagonal factor is solved by entrywise division, whose - * intermediates are bounded by the factor's conditioning exactly like the - * substitutions of the LU paths (and each division saturates per entry, like - * every step of a substitution). */ +template <typename Factor> +struct kron_factor_ops<Factor, kKronIdentityFactor> { + // Identity factor, possibly rectangular: the m x n matrix I with I(i,i) = 1 + // for i < min(m, n). I X keeps the leading min(m, n) rows of X, X I^T its + // leading min(m, n) columns. + using Scalar = typename Factor::Scalar; + using TransposedFactor = kron_identity_factor<Scalar, Factor::ColsAtCompileTime, Factor::RowsAtCompileTime>; + using InverseFactor = Factor; + static constexpr bool StoresAllEntries = false; + + static void prepare(Factor&) {} + static bool coeffIfStored(const Factor&, Index row, Index col, Scalar& value) { + if (row != col) return false; + value = Scalar(1); + return true; + } + template <typename Visitor> + static void forEachNonZero(const Factor& f, Visitor&& visit) { + for (Index k = 0; k < numext::mini(f.rows(), f.cols()); ++k) visit(k, k, Scalar(1)); + } + static Matrix<Index, Dynamic, 1> innerNonZeros(const Factor& f, bool rowMajor) { + Matrix<Index, Dynamic, 1> counts = Matrix<Index, Dynamic, 1>::Zero(rowMajor ? f.rows() : f.cols()); + counts.head(numext::mini(f.rows(), f.cols())).setOnes(); + return counts; + } + static TransposedFactor transposed(const Factor& f) { return TransposedFactor(f.cols(), f.rows()); } + static const Factor& conjugated(const Factor& f) { return f; } + static TransposedFactor adjointed(const Factor& f) { return transposed(f); } + static const Factor& inversed(const Factor& f) { return f; } + static typename Factor::PlainObject denseFactor(const Factor& f) { return blockOperand(f); } + static typename Factor::PlainObject::IdentityReturnType blockOperand(const Factor& f) { + return Factor::PlainObject::Identity(f.rows(), f.cols()); + } + static bool isSquareIdentity(const Factor& f) { return f.rows() == f.cols(); } + template <typename Dst, typename Alpha, typename Xpr> + static void addLeftProduct(Dst& dst, const Alpha& alpha, const Factor& f, const Xpr& X) { + const Index k = numext::mini(f.rows(), f.cols()); + dst.topRows(k) += alpha * X.topRows(k); + } + template <typename Dst, typename Alpha, typename Xpr, typename Work> + static void addRightProduct(Dst& dst, const Alpha& alpha, const Xpr& X, const Factor& f, Work&) { + const Index k = numext::mini(f.rows(), f.cols()); + dst.leftCols(k) += alpha * X.leftCols(k); + } + static Scalar balancedDet(const Factor&, Index&) { return Scalar(1); } +}; + +template <typename LhsMatrix, typename RhsMatrix> +struct kron_factor_ops<KroneckerOperator<LhsMatrix, RhsMatrix>, kKronKroneckerFactor> { + // Nested Kronecker factor K = L (x) R: every operation recurses into L and R. + using Factor = KroneckerOperator<LhsMatrix, RhsMatrix>; + using Scalar = typename Factor::Scalar; + using LhsOps = kron_factor_ops<LhsMatrix>; + using RhsOps = kron_factor_ops<RhsMatrix>; + using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; + using TransposedFactor = KroneckerOperator<typename LhsOps::TransposedFactor, typename RhsOps::TransposedFactor>; + using InverseFactor = KroneckerOperator<typename LhsOps::InverseFactor, typename RhsOps::InverseFactor>; + static constexpr bool StoresAllEntries = LhsOps::StoresAllEntries && RhsOps::StoresAllEntries; + + static void prepare(Factor&) {} + static bool coeffIfStored(const Factor& f, Index row, Index col, Scalar& value) { + const Index m2 = f.rhs().rows(), n2 = f.rhs().cols(); + Scalar a, b; + if (!LhsOps::coeffIfStored(f.lhs(), row / m2, col / n2, a) || + !RhsOps::coeffIfStored(f.rhs(), row % m2, col % n2, b)) + return false; + value = a * b; + return true; + } + // Visits (iA m2 + iB, jA n2 + jB) with the loop over L outermost, so a fixed + // column is visited in increasing row and a fixed row in increasing column, + // the order the sparse materialization inserts in. + template <typename Visitor> + static void forEachNonZero(const Factor& f, Visitor&& visit) { + const Index m2 = f.rhs().rows(), n2 = f.rhs().cols(); + LhsOps::forEachNonZero(f.lhs(), [&f, &visit, m2, n2](Index iA, Index jA, const Scalar& a) { + RhsOps::forEachNonZero(f.rhs(), [&visit, m2, n2, iA, jA, &a](Index iB, Index jB, const Scalar& b) { + visit(iA * m2 + iB, jA * n2 + jB, a * b); + }); + }); + } + static Matrix<Index, Dynamic, 1> innerNonZeros(const Factor& f, bool rowMajor) { + const Matrix<Index, Dynamic, Dynamic, ColMajor> counts = + RhsOps::innerNonZeros(f.rhs(), rowMajor) * LhsOps::innerNonZeros(f.lhs(), rowMajor).transpose(); + return counts.reshaped(); + } + static TransposedFactor transposed(const Factor& f) { return f.transpose(); } + static Factor conjugated(const Factor& f) { return f.conjugate(); } + static TransposedFactor adjointed(const Factor& f) { return f.adjoint(); } + static InverseFactor inversed(const Factor& f) { return f.inverse(); } + static DenseMatrix denseFactor(const Factor& f) { return DenseMatrix(f); } + static DenseMatrix blockOperand(const Factor& f) { return denseFactor(f); } + static bool isSquareIdentity(const Factor&) { return false; } + template <typename Dst, typename Alpha, typename Xpr> + static void addLeftProduct(Dst& dst, const Alpha& alpha, const Factor& f, const Xpr& X) { + f.addProduct(dst, X, alpha); + } + // X K^T without forming X^T: with X_j = X(:, j nR : (j+1) nR - 1), j < nL, + // X (L (x) R)^T = W L^T, W(:, j) = vec(X_j R^T), W (p mR) x nL in work, + // and W = X_{[p nR x nL]} when R is a square identity. + template <typename Dst, typename Alpha, typename Xpr, typename Work> + static void addRightProduct(Dst& dst, const Alpha& alpha, const Xpr& X, const Factor& f, Work& work) { + const Index p = X.rows(), mL = f.lhs().rows(), nL = f.lhs().cols(), mR = f.rhs().rows(), nR = f.rhs().cols(); + auto dstL = dst.reshaped(p * mR, mL); + if (RhsOps::isSquareIdentity(f.rhs())) { + LhsOps::addRightProduct(dstL, alpha, X.reshaped(p * nR, nL), f.lhs(), work); + } else if (LhsOps::isSquareIdentity(f.lhs())) { + for (Index j = 0; j < nL; ++j) { + auto dstj = dst.middleCols(j * mR, mR); + RhsOps::addRightProduct(dstj, alpha, X.middleCols(j * nR, nR), f.rhs(), work); + } + } else { + Work inner; // scratch for L and R while work holds W; allocated only if one is nested + work.setZero(p * mR, nL); + for (Index j = 0; j < nL; ++j) { + auto Wj = work.col(j).reshaped(p, mR); + RhsOps::addRightProduct(Wj, Alpha(1), X.middleCols(j * nR, nR), f.rhs(), inner); + } + LhsOps::addRightProduct(dstL, alpha, work, f.lhs(), inner); + } + } + static Scalar balancedDet(const Factor& f, Index& exponent) { return f.balancedDeterminant(exponent); } +}; + +/** \internal Per-factor solve adapter for KroneckerOperator::solve(): a dense + * factor is LU-factorized once, a sparse factor once by SparseLU, a diagonal + * factor is solved by entrywise division, an identity not at all, and a nested + * Kronecker factor through the adapters of its own factors. */ template <typename Factor, int Kind = kron_factor_kind<Factor>()> class kron_factor_solver { public: using Scalar = typename Factor::Scalar; using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; - explicit kron_factor_solver(const Factor& f) : m_exponent(structured_exponent_bound(f)) { - DenseMatrix normalized = f; - kron_ldexp_entries(normalized, -m_exponent); - m_lu.compute(normalized); - } - int exponent() const { return m_exponent; } - /** \internal \returns \f$ F_{norm}^{-1} M \f$. */ + explicit kron_factor_solver(const Factor& f) : m_lu(f) {} + /** \internal \returns \f$ F^{-1} M \f$. */ template <typename Xpr> DenseMatrix solveLeft(const Xpr& M) const { return m_lu.solve(M); } - /** \internal \returns \f$ M F_{norm}^{-T} \f$. */ + /** \internal \returns \f$ M F^{-T} \f$. */ template <typename Xpr> DenseMatrix solveTransposedRight(const Xpr& M) const { return m_lu.solve(M.transpose()).transpose(); } private: - int m_exponent; PartialPivLU<DenseMatrix> m_lu; }; @@ -359,11 +545,7 @@ using Scalar = typename Factor::Scalar; using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; - explicit kron_factor_solver(const Factor& f) - : m_exponent(structured_exponent_bound(f.diagonal())), m_d(f.diagonal()) { - kron_ldexp_entries(m_d, -m_exponent); - } - int exponent() const { return m_exponent; } + explicit kron_factor_solver(const Factor& f) : m_d(f.diagonal()) {} template <typename Xpr> DenseMatrix solveLeft(const Xpr& M) const { return (M.array().colwise() / m_d.array()).matrix(); @@ -374,7 +556,6 @@ } private: - int m_exponent; typename Factor::DiagonalVectorType m_d; }; @@ -385,17 +566,10 @@ using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; using ColMajorFactor = SparseMatrix<Scalar, ColMajor, typename Factor::StorageIndex>; - explicit kron_factor_solver(const Factor& f) : m_exponent(structured_exponent_bound(f.coeffs())) { - ColMajorFactor normalized(f); - normalized.makeCompressed(); - if (m_exponent != 0) { - auto values = normalized.coeffs(); - kron_ldexp_entries(values, -m_exponent); - } - m_lu.compute(normalized); + explicit kron_factor_solver(const Factor& f) { + m_lu.compute(ColMajorFactor(f)); m_factorized = m_lu.info() == Success; // else the solves fill with NaN, see kron_factor_ops::nanMatrix } - int exponent() const { return m_exponent; } template <typename Xpr> DenseMatrix solveLeft(const Xpr& M) const { if (!m_factorized) return kron_factor_ops<Factor>::nanMatrix(M.rows(), M.cols()); @@ -411,11 +585,122 @@ } private: - int m_exponent; bool m_factorized; SparseLU<ColMajorFactor> m_lu; }; +template <typename Factor> +class kron_factor_solver<Factor, kKronIdentityFactor> { + public: + using Scalar = typename Factor::Scalar; + using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; + + explicit kron_factor_solver(const Factor&) {} + template <typename Xpr> + DenseMatrix solveLeft(const Xpr& M) const { + return M; + } + template <typename Xpr> + DenseMatrix solveTransposedRight(const Xpr& M) const { + return M; + } +}; + +/** \internal A nested factor K = L (x) R is solved through the solvers of L and + * R, each factorized once. */ +template <typename LhsMatrix, typename RhsMatrix> +class kron_factor_solver<KroneckerOperator<LhsMatrix, RhsMatrix>, kKronKroneckerFactor> { + public: + using Factor = KroneckerOperator<LhsMatrix, RhsMatrix>; + using Scalar = typename Factor::Scalar; + using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; + + explicit kron_factor_solver(const Factor& f) + : m_solverA(squareFactor(f.lhs())), + m_solverB(squareFactor(f.rhs())), + m_n1(f.lhs().cols()), + m_n2(f.rhs().cols()) {} + template <typename Xpr> + DenseMatrix solveLeft(const Xpr& M) const { + DenseMatrix x(M.rows(), M.cols()); + Factor::solveWith(m_solverA, m_solverB, m_n1, m_n2, M, x); + return x; + } + // M K^{-T} = (K^{-1} M^T)^T. + template <typename Xpr> + DenseMatrix solveTransposedRight(const Xpr& M) const { + return solveLeft(M.transpose()).transpose(); + } + + private: + template <typename F> + static const F& squareFactor(const F& f) { + eigen_assert(f.rows() == f.cols() && "KroneckerOperator::solve requires square factors"); + return f; + } + + kron_factor_solver<LhsMatrix> m_solverA; + kron_factor_solver<RhsMatrix> m_solverB; + Index m_n1, m_n2; +}; + +/** \internal Per-factor eigen- and singular-value decompositions behind + * KroneckerOperator::eigenvalues(), eigenvectors(), singularValues(), matrixU() + * and matrixV(): a dense decomposition of the materialized factor, except for a + * nested Kronecker factor, which recurses into its own factors -- O(n_L^3 + + * n_R^3) instead of O((n_L n_R)^3), and the eigenvalues of defective factors + * keep the sensitivity of the factors' Jordan blocks instead of the longer + * blocks of their product. */ +template <typename Factor, int Kind = kron_factor_kind<Factor>()> +struct kron_factor_spectrum { + using Ops = kron_factor_ops<Factor>; + using Scalar = typename Factor::Scalar; + using RealScalar = typename NumTraits<Scalar>::Real; + using ComplexScalar = std::complex<RealScalar>; + using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; + using RealVector = Matrix<RealScalar, Dynamic, 1>; + using ComplexMatrix = Matrix<ComplexScalar, Dynamic, Dynamic, ColMajor>; + using ComplexVector = Matrix<ComplexScalar, Dynamic, 1>; + using Eigenvectors = ComplexMatrix; + using SingularVectors = DenseMatrix; + + static ComplexMatrix complexFactor(const Factor& f) { return Ops::denseFactor(f).template cast<ComplexScalar>(); } + static ComplexVector eigenvalues(const Factor& f) { + ComplexEigenSolver<ComplexMatrix> es(complexFactor(f), /*computeEigenvectors=*/false); + eigen_assert(es.info() == Success); + return es.eigenvalues(); + } + static Eigenvectors eigenvectors(const Factor& f) { + ComplexEigenSolver<ComplexMatrix> es(complexFactor(f)); + eigen_assert(es.info() == Success); + return es.eigenvectors(); + } + static RealVector singularValues(const Factor& f) { + return BDCSVD<DenseMatrix>(Ops::denseFactor(f)).singularValues(); + } + static SingularVectors matrixU(const Factor& f) { + return BDCSVD<DenseMatrix, ComputeThinU>(Ops::denseFactor(f)).matrixU(); + } + static SingularVectors matrixV(const Factor& f) { + return BDCSVD<DenseMatrix, ComputeThinV>(Ops::denseFactor(f)).matrixV(); + } +}; + +template <typename LhsMatrix, typename RhsMatrix> +struct kron_factor_spectrum<KroneckerOperator<LhsMatrix, RhsMatrix>, kKronKroneckerFactor> { + using Factor = KroneckerOperator<LhsMatrix, RhsMatrix>; + using Eigenvectors = KroneckerOperator<typename kron_factor_spectrum<LhsMatrix>::Eigenvectors, + typename kron_factor_spectrum<RhsMatrix>::Eigenvectors>; + using SingularVectors = KroneckerOperator<typename kron_factor_spectrum<LhsMatrix>::SingularVectors, + typename kron_factor_spectrum<RhsMatrix>::SingularVectors>; + + static typename Factor::ComplexVector eigenvalues(const Factor& f) { return f.eigenvalues(); } + static Eigenvectors eigenvectors(const Factor& f) { return f.eigenvectors(); } + static typename Factor::RealVector singularValues(const Factor& f) { return f.singularValues(); } + static SingularVectors matrixU(const Factor& f) { return f.matrixU(); } + static SingularVectors matrixV(const Factor& f) { return f.matrixV(); } +}; + } // namespace internal /** \ingroup StructuredMatrices_Module @@ -425,19 +710,31 @@ * * For \c A of size \c m1 x \c n1 and \c B of size \c m2 x \c n2, the Kronecker * product is the \c m1*m2 x \c n1*n2 block matrix whose block \c (i,j) is - * \c A(i,j)*B. This class stores only the two factors and evaluates every - * operation through them: + * \c A(i,j)*B. * - * - the matrix-vector product uses the vec identity - * \f$ (A \otimes B)\,\mathrm{vec}(X) = \mathrm{vec}(B X A^T) \f$, costing - * O(m2 n2 n1 + m2 n1 m1) instead of the O(m1 m2 n1 n2) of a materialized - * product; + * Throughout, \f$ \mathrm{vec}(X) \f$ stacks the \b columns of \c X, whatever + * the storage order of the operands and independently of + * \c EIGEN_DEFAULT_TO_ROW_MAJOR: for \c X of size \c n2 x \c n1, entry + * \c j*n2+i of \f$ \mathrm{vec}(X) \f$ is \c X(i,j), and its inverse + * \f$ \mathrm{mat}(x) \f$ is \c x.reshaped(n2,n1). Under this convention the + * product is \f$ (A \otimes B)\,\mathrm{vec}(X) = \mathrm{vec}(B X A^T) \f$; with + * row-major stacking \f$ \mathrm{vec}_r \f$ of \c Z of size \c n1 x \c n2 the + * same product reads \f$ (A \otimes B)\,\mathrm{vec}_r(Z) = \mathrm{vec}_r(A Z B^T) \f$. + * + * This class stores only the two factors and evaluates every operation through + * them: + * + * - the product uses the vec identity, costing O(m2 n2 n1 + m2 n1 m1) per + * right-hand side instead of the O(m1 m2 n1 n2) of a materialized product; + * right-hand sides are applied in cache-sized batches, one product with each + * factor per batch; * - linear solves and \ref inverse factor through decompositions of \c A and - * \c B (\f$ (A \otimes B)^{-1} = A^{-1} \otimes B^{-1} \f$); \ref rank and - * minimum-norm least-squares solves (\f$ (A \otimes B)^+ = A^+ \otimes B^+ \f$) - * go through the factor SVDs, thresholding the pairwise singular-value - * products \f$ \sigma_i(A)\,\sigma_j(B) \f$ -- the singular values of the - * Kronecker product -- at the product level; + * \c B (\f$ (A \otimes B)^{-1} = A^{-1} \otimes B^{-1} \f$); minimum-norm + * least-squares solves (\f$ (A \otimes B)^+ = A^+ \otimes B^+ \f$) use one + * complete orthogonal decomposition per factor, deciding each factor's rank + * on its own; \ref rank goes through the factor SVDs, thresholding the + * pairwise singular-value products \f$ \sigma_i(A)\,\sigma_j(B) \f$ -- the + * singular values of the Kronecker product -- at the product level; * - the eigendecomposition and the (thin) SVD are Kronecker products of the * factor decompositions: the eigenvector and singular-vector matrices are * returned as \c KroneckerOperator objects themselves, never materialized; @@ -460,52 +757,75 @@ * an operator meant to be applied and solved with, without ever forming the * product. * - * Either factor may be a \c DiagonalMatrix, with the identity as the - * unit-diagonal special case. A diagonal factor is stored as its diagonal -- - * O(n) instead of O(n^2) -- its side of every product is a diagonal scaling - * instead of a GEMM, \ref solve divides entrywise instead of factorizing, and - * \ref transpose, \ref conjugate, \ref adjoint, \ref inverse and - * \ref determinant never leave diagonal form. This covers the - * identity-Kronecker operators \f$ I \otimes A \f$ and \f$ A \otimes I \f$ - * ubiquitous in finite-difference and Sylvester/Lyapunov settings, e.g. + * Either factor may be a \c DiagonalMatrix. A diagonal factor is stored as its + * diagonal -- O(n) instead of O(n^2) -- its side of every product is a diagonal + * scaling instead of a GEMM, \ref solve divides entrywise instead of + * factorizing, and \ref transpose, \ref conjugate, \ref adjoint, \ref inverse + * and \ref determinant never leave diagonal form. + * + * Either factor may also be an identity, passed as an \c Identity() expression + * (\c MatrixXd::Identity(p, p)): makeKroneckerOperator() stores it as an + * internal identity factor holding only its dimensions (assignable, unlike the + * expression), its side of a product is skipped, and its solves are the + * identity map. This covers the identity-Kronecker operators + * \f$ I \otimes A \f$ and \f$ A \otimes I \f$ ubiquitous in finite-difference + * and Sylvester/Lyapunov settings, e.g. * \code - * auto K = makeKroneckerOperator(VectorXd::Ones(p).asDiagonal(), A); // I_p (x) A, applied in O(p m n) + * auto K = makeKroneckerOperator(MatrixXd::Identity(p, p), A); // I_p (x) A, y = K * x is one product with A * \endcode + * A rectangular \c Identity(m, n) is the \c m x \c n matrix with ones on the + * main diagonal. The unit \c DiagonalMatrix \c VectorXd::Ones(p).asDiagonal() + * describes the same operator but is applied as a scaling. + * * The decomposition family (\ref eigenvalues, \ref eigenvectors, * \ref singularValues, \ref matrixU, \ref matrixV, \ref leastSquaresSolve, - * \ref rank) currently materializes a diagonal factor densely for the factor - * decomposition. + * \ref rank) currently materializes a diagonal or identity factor densely for + * the factor decomposition. * - * Either factor may also be a \c SparseMatrix, stored compressed. For finite - * inputs, its side of a product is sparse-dense -- O(nnz) per right-hand-side column - * instead of O(m n) -- \ref solve factorizes it once with \c SparseLU and - * back-substitutes per column, \ref transpose, \ref conjugate and \ref adjoint + * Either factor may also be a \c SparseMatrix, stored compressed. Its side of + * a product is a sparse-dense product -- O(nnz) instead of O(m n) per column of + * the reshaped right-hand side -- \ref solve factorizes it once with + * \c SparseLU and back-substitutes, \ref transpose, \ref conjugate and \ref adjoint * stay sparse, and \ref determinant accumulates the SparseLU pivots in the same * balanced form as the dense LU path, scaling small factors up before * factorization to keep the elimination out of the subnormal range. Its \ref inverse is * dense (one SparseLU solve against the identity), and the decomposition family - * densifies it like a diagonal factor. As in every sparse kernel, absent entries - * are exact zeros: a structural zero annihilates the Inf or NaN it meets, where a stored zero - * (or a dense factor) would produce NaN. Products with non-finite inputs visit - * pairs of stored factor entries to preserve this distinction without - * materializing the product. A sparse factor whose \c SparseLU factorization + * densifies it like a diagonal factor. A product with non-finite data + * propagates Inf/NaN as the two factor products do, not as a product with the + * materialized sparse matrix would. A sparse factor whose \c SparseLU factorization * fails -- an exactly zero or non-finite pivot column -- solves * and inverts to NaN, and has determinant 0 when exactly singular (NaN when * non-finite). Assigning the operator to a \c SparseMatrix materializes the * product sparsely, every inner vector reserved to its exact size. For a sparse * \c A of size \c n with \c nnz(A) stored entries: * \code - * auto K = makeKroneckerOperator(VectorXd::Ones(p).asDiagonal(), A); // I_p (x) A + * auto K = makeKroneckerOperator(MatrixXd::Identity(p, p), A); // I_p (x) A * VectorXd y = K * x; // O(p nnz(A)), no p n x p n matrix * VectorXd z = K.solve(b); // one SparseLU of A, then p column solves * SparseMatrix<double> M; * M = K; // p nnz(A) stored entries, when the matrix itself is needed * \endcode * - * \tparam LhsMatrix the plain type of the left factor \c A: a dense \c Matrix, - * a \c DiagonalMatrix to exploit diagonal structure, or a - * \c SparseMatrix to exploit sparsity. - * \tparam RhsMatrix the plain type of the right factor \c B, under the same + * Finally, either factor may be a \c KroneckerOperator itself, so products of + * three or more factors stay implicit: \f$ A \otimes (B \otimes C) \f$ applies + * \f$ B \otimes C \f$ through its own vec identity, and solves, the + * determinant, the transposition family, materialization, \ref eigenvalues, + * \ref eigenvectors and the SVD (\ref singularValues, \ref matrixU, + * \ref matrixV, whose vector matrices nest the same way) recurse into the + * nested factors; \ref rank and \ref leastSquaresSolve materialize a nested + * factor. \c makeKroneckerOperator(a, b, c, ...) builds this right-nested + * form; e.g. the middle term \f$ I_p \otimes A \otimes I_q \f$ of a 3-D + * finite-difference operator is + * \code + * auto K = makeKroneckerOperator(MatrixXd::Identity(p, p), A, MatrixXd::Identity(q, q)); + * VectorXd y = K * x; // O(p q nnz(A)), A (x) I_q is never formed either + * \endcode + * + * \tparam LhsMatrix the type of the left factor \c A: a dense \c Matrix, a + * \c DiagonalMatrix to exploit diagonal structure, a \c SparseMatrix to + * exploit sparsity, an identity factor (what makeKroneckerOperator() + * stores an \c Identity() expression as), or a \c KroneckerOperator. + * \tparam RhsMatrix the type of the right factor \c B, under the same * convention; its scalar type must match that of \c LhsMatrix. * * \sa makeKroneckerOperator(), class Circulant, class Toeplitz @@ -516,36 +836,39 @@ using Scalar = typename LhsMatrix::Scalar; using RealScalar = typename NumTraits<Scalar>::Real; using StorageIndex = int; + + static_assert(std::is_same<Scalar, typename RhsMatrix::Scalar>::value, + "KroneckerOperator requires both factors to have the same scalar type"); + static_assert((internal::kron_factor_is_dense_matrix<LhsMatrix>::value || + internal::kron_factor_kind<LhsMatrix>() != internal::kKronDenseFactor) && + (internal::kron_factor_is_dense_matrix<RhsMatrix>::value || + internal::kron_factor_kind<RhsMatrix>() != internal::kKronDenseFactor), + "KroneckerOperator factors must be plain Matrix, DiagonalMatrix or SparseMatrix types, identity " + "factors (makeKroneckerOperator stores an Identity() expression as one) or KroneckerOperators (owning " + "their storage: views and other expressions would dangle)"); + + private: + // Factor-kind dispatch, see kron_factor_ops. + using LhsOps = internal::kron_factor_ops<LhsMatrix>; + using RhsOps = internal::kron_factor_ops<RhsMatrix>; + using LhsSpectrum = internal::kron_factor_spectrum<LhsMatrix>; + using RhsSpectrum = internal::kron_factor_spectrum<RhsMatrix>; + template <typename, int> + friend struct internal::kron_factor_ops; + template <typename, int> + friend class internal::kron_factor_solver; + + public: using ComplexScalar = std::complex<RealScalar>; // The vec-trick reshapes below identify a vector of length n1*n2 with an // n2 x n1 matrix whose columns are stacked, so every workspace taking part // in a reshape is pinned to ColMajor explicitly: the semantics must not // change under EIGEN_DEFAULT_TO_ROW_MAJOR. using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; - using DenseVector = Matrix<Scalar, Dynamic, 1>; using RealVector = Matrix<RealScalar, Dynamic, 1>; using ComplexMatrix = Matrix<ComplexScalar, Dynamic, Dynamic, ColMajor>; using ComplexVector = Matrix<ComplexScalar, Dynamic, 1>; - static_assert(std::is_same<Scalar, typename RhsMatrix::Scalar>::value, - "KroneckerOperator requires both factors to have the same scalar type"); - static_assert((internal::kron_factor_is_dense_matrix<LhsMatrix>::value || - internal::kron_factor_is_diagonal<LhsMatrix>::value || - internal::kron_factor_is_sparse_matrix<LhsMatrix>::value) && - (internal::kron_factor_is_dense_matrix<RhsMatrix>::value || - internal::kron_factor_is_diagonal<RhsMatrix>::value || - internal::kron_factor_is_sparse_matrix<RhsMatrix>::value), - "KroneckerOperator factors must be plain Matrix, DiagonalMatrix or SparseMatrix types (owning their " - "storage: views and expressions would dangle)"); - - private: - // Factor-kind dispatch (dense, diagonal or sparse), see kron_factor_ops. - using LhsOps = internal::kron_factor_ops<LhsMatrix>; - using RhsOps = internal::kron_factor_ops<RhsMatrix>; - static constexpr bool kHasSparseFactor = internal::kron_factor_is_sparse_matrix<LhsMatrix>::value || - internal::kron_factor_is_sparse_matrix<RhsMatrix>::value; - - public: static constexpr int RowsAtCompileTime = internal::size_at_compile_time(LhsMatrix::RowsAtCompileTime, RhsMatrix::RowsAtCompileTime); static constexpr int ColsAtCompileTime = @@ -562,7 +885,8 @@ /** Builds the operator \c A (x) \c B from the two factors, evaluated into the * operator's factor types: a dense expression into a \c Matrix, a diagonal * one into a \c DiagonalMatrix (stored as its diagonal), a sparse one into a - * compressed \c SparseMatrix. */ + * compressed \c SparseMatrix, an \c Identity() expression into an identity + * factor holding its dimensions; a \c KroneckerOperator is copied as it is. */ template <typename LhsDerived, typename RhsDerived> KroneckerOperator(const EigenBase<LhsDerived>& a, const EigenBase<RhsDerived>& b) : m_A(a.derived()), m_B(b.derived()) { @@ -609,24 +933,13 @@ * factor (a diagonal factor is solved by entrywise division instead): * reshaping \c b column-wise as \c mat(b) of size \c n2 x \c n1, the system * reads \f$ B X A^T = \mathrm{mat}(b) \f$, so - * \f$ X = B^{-1} \mathrm{mat}(b) A^{-T} \f$. Supports multiple right-hand - * sides at O(n1^3 + n2^3 + nrhs (n1 + n2) n1 n2) total cost for dense - * factors; a diagonal factor contributes only O(nrhs n1 n2) per application, - * a sparse one its factorization plus \c nrhs times its column solves. - * - * The solves run in a normalized frame [2][3]: the factors and each right-hand - * side are rescaled to unit magnitude by exact powers of two, and the combined - * exponent is folded back into the solution entrywise afterwards. In the raw - * frame the intermediate \f$ B^{-1} \mathrm{mat}(b) \f$ can overflow, or - * silently underflow to zero, on factor magnitudes alone even when the - * solution is representable (the subsequent \f$ A^{-T} \f$ rescales it: think - * \c B huge and \c A tiny); in the normalized frame the intermediates are - * bounded by the conditioning of the factors, and the final fold saturates to - * 0/Inf exactly when the true solution leaves the representable range. - * Partial pivoting is invariant under the uniform power-of-two normalization - * and every substitution step scales exactly with it, so the result is - * bit-identical to the unnormalized evaluation whenever that evaluation - * encounters no intermediate over- or underflow. + * \f$ X = B^{-1} \mathrm{mat}(b) A^{-T} \f$. Right-hand sides are solved + * in cache-sized batches, one solve per factor per batch, at O(n1^3 + n2^3 + + * nrhs (n1 + n2) n1 n2) total cost for dense factors; a diagonal factor + * contributes only O(nrhs n1 n2), a sparse one its factorization plus its + * substitutions. As in any chain of solves, \f$ B^{-1} \mathrm{mat}(b) \f$ + * can overflow (\c B tiny) or underflow (\c B huge, \c A tiny) on extreme + * factor magnitudes even when the solution is representable. * \warning Both factors must be invertible, like in \c PartialPivLU: a * singular dense factor substitutes Inf/NaN through the solution, and a * sparse factor whose \c SparseLU factorization fails solves to NaN. Use @@ -639,90 +952,66 @@ const Index n1 = m_A.cols(), n2 = m_B.cols(); eigen_assert(m_A.rows() == n1 && m_B.rows() == n2 && "KroneckerOperator::solve requires square factors"); eigen_assert(b.rows() == n1 * n2 && "right-hand side has the wrong number of rows"); - // The exponent bounds are 0 for zero or non-finite data, which is then left - // unnormalized (Inf/NaN propagate through the substitutions as usual). const internal::kron_factor_solver<LhsMatrix> solverA(m_A); const internal::kron_factor_solver<RhsMatrix> solverB(m_B); Matrix<Scalar, ColsAtCompileTime, Rhs::ColsAtCompileTime> x(n1 * n2, b.cols()); - DenseVector bc(n1 * n2); - DenseMatrix X(n2, n1); - for (Index k = 0; k < b.cols(); ++k) { - bc = b.col(k); - const int ec = internal::structured_exponent_bound(bc); - if (ec != 0) internal::kron_ldexp_entries(bc, -ec); - X = solverA.solveTransposedRight(solverB.solveLeft(bc.reshaped(n2, n1))); - // Fold the combined exponent back. The entrywise ldexp saturates exactly - // where the true solution over- or underflows; a multiplicative fold could - // not (the combined exponent can exceed the representable range of any - // fixed number of power-of-two factors). - const int e = ec - solverA.exponent() - solverB.exponent(); - if (e != 0) internal::kron_ldexp_entries(X, e); - x.col(k) = X.reshaped(); - } + solveWith(solverA, solverB, n1, n2, b.derived(), x); return x; } /** \returns the minimum-norm least-squares solution of \c (*this) * x = b, - * through one SVD per factor: with \f$ A = U_A \Sigma_A V_A^H \f$ and - * \f$ B = U_B \Sigma_B V_B^H \f$, the SVD of the product is - * \f$ (U_A \otimes U_B)(\Sigma_A \otimes \Sigma_B)(V_A \otimes V_B)^H \f$, and - * the pseudo-inverse is applied via the vec identity. The singular values of - * the product are the pairwise products \f$ \sigma_i(A)\,\sigma_j(B) \f$, so - * the truncation thresholds those products against the product-level - * threshold of \ref rank, in the same overflow-safe ratio form -- factor-wise - * truncation would invert modes that are negligible at the product level. - * Handles rectangular and rank-deficient factors. Supports multiple - * right-hand sides. + * from one complete orthogonal decomposition per factor. Since + * \f$ (A \otimes B)^+ = A^+ \otimes B^+ \f$, the solution is + * \f$ X = B^+ \mathrm{mat}(b)\,(A^+)^T \f$, applied as one multi-right-hand-side + * solve with each factor's decomposition per batch, no SVD needed. Handles + * rectangular and rank-deficient factors. Supports multiple right-hand sides, + * applied in cache-sized batches. * - * Unlike \ref solve, no rescaling of the intermediates is needed here: the - * projection and back-transformation matrices have orthonormal columns, so no - * intermediate can outgrow the data by more than a dimension factor, exactly - * as in a dense product. */ + * The numerical rank is decided per factor, from the diagonal of the pivoted + * QR inside \c CompleteOrthogonalDecomposition (so, as for any column-pivoted + * QR, a near-deficiency of the Kahan type can go undetected). This is + * backward stable: the decompositions perturb \c A and \c B separately, so + * each product singular value \f$ \sigma_i(A)\,\sigma_j(B) \f$ inherits only + * its factors' relative errors. It therefore need not agree with \ref rank, + * which thresholds the pairwise singular-value products at the product level: + * factors that are each full rank can form modes that \ref rank drops but this + * method inverts. + * + * Each factor is scaled by an exact power of two before its decomposition, so + * factor magnitudes alone cannot over- or underflow the decompositions or the + * intermediates; the right-hand side is not rescaled. A non-finite factor + * solves to NaN. */ template <typename Rhs> Matrix<Scalar, ColsAtCompileTime, Rhs::ColsAtCompileTime> leastSquaresSolve(const MatrixBase<Rhs>& b) const { EIGEN_STATIC_ASSERT(RowsAtCompileTime == Dynamic || Rhs::RowsAtCompileTime == Dynamic || int(RowsAtCompileTime) == int(Rhs::RowsAtCompileTime), YOU_MIXED_MATRICES_OF_DIFFERENT_SIZES) - const Index m1 = m_A.rows(), m2 = m_B.rows(); + const Index m1 = m_A.rows(), m2 = m_B.rows(), n1 = m_A.cols(), n2 = m_B.cols(), r = b.cols(); eigen_assert(b.rows() == m1 * m2 && "right-hand side has the wrong number of rows"); - BDCSVD<DenseMatrix, ComputeThinU | ComputeThinV> svdA(LhsOps::denseFactor(m_A)), svdB(RhsOps::denseFactor(m_B)); - const RealVector sa = svdA.singularValues(), sb = svdB.singularValues(); - const RealScalar tol = relativeRankThreshold(); - const Index kA = sa.size(), kB = sb.size(); - Matrix<Scalar, ColsAtCompileTime, Rhs::ColsAtCompileTime> x(cols(), b.cols()); - if (sa[0] == RealScalar(0) || sb[0] == RealScalar(0)) { - // An exactly zero factor zeroes the whole operator, whose pseudo-inverse is - // zero (and the singular-value ratios below would be 0/0). - x.setZero(); + Matrix<Scalar, ColsAtCompileTime, Rhs::ColsAtCompileTime> x(cols(), r); + // The pivoted QR squares column norms, which over- or underflow on extreme + // factor magnitudes, so decompose 2^-eA A and 2^-eB B and fold + // 2^-(eA+eB) back into the solution. + DenseMatrix An = LhsOps::denseFactor(m_A), Bn = RhsOps::denseFactor(m_B); + if (!An.allFinite() || !Bn.allFinite()) { + // The pivoted QR would rank a non-finite column out instead of propagating it. + x.setConstant(Scalar(NumTraits<RealScalar>::quiet_NaN())); return x; } - const SingularModes modesA(sa), modesB(sb); - Matrix<RealScalar, Dynamic, Dynamic, ColMajor> divisors(kB, kA); - Matrix<int, Dynamic, Dynamic, ColMajor> exponents(kB, kA); - Matrix<bool, Dynamic, Dynamic, ColMajor> retained(kB, kA); - for (Index j = 0; j < kA; ++j) - for (Index i = 0; i < kB; ++i) { - retained(i, j) = modesA.retains(modesB, j, i, tol); - divisors(i, j) = modesA.mantissas[j] * modesB.mantissas[i]; - exponents(i, j) = -(modesA.exponents[j] + modesB.exponents[i]); - } - DenseVector bc(m1 * m2); - DenseMatrix M(kB, kA), X(m_B.cols(), m_A.cols()); - for (Index k = 0; k < b.cols(); ++k) { - bc = b.col(k); - // By [1], M = U_B^H mat(b) conj(U_A) matricizes (U_A (x) U_B)^H b. - M.noalias() = svdB.matrixU().adjoint() * bc.reshaped(m2, m1) * svdA.matrixU().conjugate(); - // Apply the exponent before dividing by the mantissa product [2]: the - // singular-value product or its reciprocal need not be representable. - for (Index j = 0; j < kA; ++j) - for (Index i = 0; i < kB; ++i) { - if (retained(i, j)) - M(i, j) = internal::structured_ldexp_clamped(M(i, j), Index(exponents(i, j))) / divisors(i, j); - else - M(i, j) = Scalar(0); - } - X.noalias() = svdB.matrixV() * M * svdA.matrixV().transpose(); - x.col(k) = X.reshaped(); + const int eA = internal::structured_exponent_bound(An), eB = internal::structured_exponent_bound(Bn); + internal::structured_ldexp_entries_exact(An, -eA); + internal::structured_ldexp_entries_exact(Bn, -eB); + const CompleteOrthogonalDecomposition<DenseMatrix> codA(An), codB(Bn); + typename internal::nested_eval<Rhs, 1>::type actualRhs(b.derived()); + DenseMatrix S, Z, X; + const Index chunk = rhsChunk<Scalar>(m1 * m2 + n2 * m1 + n1 * n2); + for (Index k0 = 0; k0 < r; k0 += chunk) { + const Index c = numext::mini(chunk, r - k0); + stackColumns(S, actualRhs.middleCols(k0, c), m2, m1); + Z = codB.solve(S.reshaped(m2, c * m1)); + X = codA.solve(Z.reshaped(n2 * c, m1).transpose()).transpose(); + internal::structured_ldexp_entries(X, -(eA + eB)); + for (Index k = 0; k < c; ++k) x.col(k0 + k).reshaped(n2, n1) = X.middleRows(k * n2, n2); } return x; } @@ -737,11 +1026,11 @@ * \f$ (\sigma_i(A)/\sigma_{max}(A))(\sigma_j(B)/\sigma_{max}(B)) \f$ against * \c min(rows(),cols()) * epsilon, and the clamp in exponent space, so that * neither the thresholds nor the products can spuriously under- or overflow. - * This is the same threshold \ref leastSquaresSolve uses to decide which - * modes to invert. Thresholding the products matters: factors that are each + * Thresholding the products matters: factors that are each * full rank against their own threshold can still form pairwise products * that are negligible at the product level, so the rank can be smaller than - * the product of the factor ranks. */ + * the product of the factor ranks. This costs two SVDs; \ref leastSquaresSolve + * decides each factor's rank on its own instead. */ Index rank() const { BDCSVD<DenseMatrix> svdA(LhsOps::denseFactor(m_A)), svdB(RhsOps::denseFactor(m_B)); const RealVector sa = svdA.singularValues(), sb = svdB.singularValues(); @@ -772,8 +1061,9 @@ /** \returns the determinant \f$ \det(A)^{n_2} \det(B)^{n_1} \f$ for square * factors \c A of size \c n1 and \c B of size \c n2. The product is * accumulated from the factor LU diagonals (the \c SparseLU pivots for a - * sparse factor, the diagonal itself for a diagonal factor, skipping the LU) - * in the balanced form \c m * 2^e -- + * sparse factor, the diagonal itself for a diagonal factor, skipping the LU; + * 1 for an identity; recursively for a nested Kronecker factor, whose own + * factors must then be square) in the balanced form \c m * 2^e -- * every factor and the running product are renormalized to unit magnitude * with the power of two tracked separately -- so the partial products (in * particular \c det(A) and \c det(B) themselves, which can overflow or @@ -783,8 +1073,7 @@ eigen_assert(m_A.rows() == m_A.cols() && m_B.rows() == m_B.cols() && "KroneckerOperator::determinant requires square factors"); Index exponent = 0; - Scalar mant = balancedDetPow(m_A, m_B.cols(), exponent); - mant = internal::structured_balance(mant * balancedDetPow(m_B, m_A.cols(), exponent), exponent); + const Scalar mant = balancedDeterminant(exponent); return internal::structured_ldexp_clamped(mant, exponent); } @@ -796,27 +1085,19 @@ ComplexVector eigenvalues() const { eigen_assert(m_A.rows() == m_A.cols() && m_B.rows() == m_B.cols() && "KroneckerOperator::eigenvalues requires square factors"); - ComplexEigenSolver<ComplexMatrix> esA(LhsOps::denseFactor(m_A).template cast<ComplexScalar>(), - /*computeEigenvectors=*/false); - ComplexEigenSolver<ComplexMatrix> esB(RhsOps::denseFactor(m_B).template cast<ComplexScalar>(), - /*computeEigenvectors=*/false); - eigen_assert(esA.info() == Success && esB.info() == Success); // Column-major stacking of the nB x nA outer product puts mu_j(B) lambda_i(A) // at index i*nB + j, the Kronecker order. - return (esB.eigenvalues() * esA.eigenvalues().transpose()).reshaped(); + return (RhsSpectrum::eigenvalues(m_B) * LhsSpectrum::eigenvalues(m_A).transpose()).reshaped(); } /** \returns the matrix of eigenvectors \f$ V_A \otimes V_B \f$ for square * factors -- itself a Kronecker operator, never materialized. Column * \c i*n2 + j is \f$ v_i(A) \otimes v_j(B) \f$ and matches * \c eigenvalues()[i*n2 + j]. Assign it to a dense matrix to materialize. */ - KroneckerOperator<ComplexMatrix, ComplexMatrix> eigenvectors() const { + KroneckerOperator<typename LhsSpectrum::Eigenvectors, typename RhsSpectrum::Eigenvectors> eigenvectors() const { eigen_assert(m_A.rows() == m_A.cols() && m_B.rows() == m_B.cols() && "KroneckerOperator::eigenvectors requires square factors"); - ComplexEigenSolver<ComplexMatrix> esA(LhsOps::denseFactor(m_A).template cast<ComplexScalar>()); - ComplexEigenSolver<ComplexMatrix> esB(RhsOps::denseFactor(m_B).template cast<ComplexScalar>()); - eigen_assert(esA.info() == Success && esB.info() == Success); - return {esA.eigenvectors(), esB.eigenvectors()}; + return {LhsSpectrum::eigenvectors(m_A), RhsSpectrum::eigenvectors(m_B)}; } /** \returns the singular values of the thin SVD @@ -827,26 +1108,23 @@ * \c U and \c V); for rectangular shapes the full SVD pads this set with * \c min(rows(),cols()) - k_A*k_B structural zeros. */ RealVector singularValues() const { - BDCSVD<DenseMatrix> svdA(LhsOps::denseFactor(m_A)), svdB(RhsOps::denseFactor(m_B)); // Column-major stacking of the kB x kA outer product puts sigma_j(B) sigma_i(A) // at index i*kB + j, the Kronecker order. - return (svdB.singularValues() * svdA.singularValues().transpose()).reshaped(); + return (RhsSpectrum::singularValues(m_B) * LhsSpectrum::singularValues(m_A).transpose()).reshaped(); } /** \returns the left singular vectors \f$ U_A \otimes U_B \f$ of the thin SVD, * itself a Kronecker operator with orthonormal columns; column \c i*k_B + j * matches \c singularValues()[i*k_B + j]. */ - KroneckerOperator<DenseMatrix, DenseMatrix> matrixU() const { - BDCSVD<DenseMatrix, ComputeThinU> svdA(LhsOps::denseFactor(m_A)), svdB(RhsOps::denseFactor(m_B)); - return {svdA.matrixU(), svdB.matrixU()}; + KroneckerOperator<typename LhsSpectrum::SingularVectors, typename RhsSpectrum::SingularVectors> matrixU() const { + return {LhsSpectrum::matrixU(m_A), RhsSpectrum::matrixU(m_B)}; } /** \returns the right singular vectors \f$ V_A \otimes V_B \f$ of the thin SVD, * itself a Kronecker operator with orthonormal columns; column \c i*k_B + j * matches \c singularValues()[i*k_B + j]. */ - KroneckerOperator<DenseMatrix, DenseMatrix> matrixV() const { - BDCSVD<DenseMatrix, ComputeThinV> svdA(LhsOps::denseFactor(m_A)), svdB(RhsOps::denseFactor(m_B)); - return {svdA.matrixV(), svdB.matrixV()}; + KroneckerOperator<typename LhsSpectrum::SingularVectors, typename RhsSpectrum::SingularVectors> matrixV() const { + return {LhsSpectrum::matrixV(m_A), RhsSpectrum::matrixV(m_B)}; } /** \internal Writes the representation into \a dst, dense or sparse according @@ -882,22 +1160,25 @@ void evalToImpl(Dest& dst, std::false_type) const { const Index m2 = m_B.rows(), n2 = m_B.cols(); if (!LhsOps::StoresAllEntries) dst.setZero(); + const auto& B = RhsOps::blockOperand(m_B); LhsOps::forEachNonZero( - m_A, [&dst, m2, n2, this](Index i, Index j, const Scalar& a) { dst.block(i * m2, j * n2, m2, n2) = a * m_B; }); + m_A, [&dst, &B, m2, n2](Index i, Index j, const Scalar& a) { dst.block(i * m2, j * n2, m2, n2) = a * B; }); } template <typename Dest> void addToImpl(Dest& dst, std::false_type) const { const Index m2 = m_B.rows(), n2 = m_B.cols(); + const auto& B = RhsOps::blockOperand(m_B); LhsOps::forEachNonZero( - m_A, [&dst, m2, n2, this](Index i, Index j, const Scalar& a) { dst.block(i * m2, j * n2, m2, n2) += a * m_B; }); + m_A, [&dst, &B, m2, n2](Index i, Index j, const Scalar& a) { dst.block(i * m2, j * n2, m2, n2) += a * B; }); } template <typename Dest> void subToImpl(Dest& dst, std::false_type) const { const Index m2 = m_B.rows(), n2 = m_B.cols(); + const auto& B = RhsOps::blockOperand(m_B); LhsOps::forEachNonZero( - m_A, [&dst, m2, n2, this](Index i, Index j, const Scalar& a) { dst.block(i * m2, j * n2, m2, n2) -= a * m_B; }); + m_A, [&dst, &B, m2, n2](Index i, Index j, const Scalar& a) { dst.block(i * m2, j * n2, m2, n2) -= a * B; }); } /** \internal The sparse representation. Every stored entry of \c A (all @@ -958,102 +1239,94 @@ } /** \internal Computes \c dst += alpha * (*this) * rhs through the vec identity - * \f$ (A \otimes B)\,\mathrm{vec}(X) = \mathrm{vec}(B X A^T) \f$ of [1], column - * by column. \c ProductScalar is the promoted scalar of the product (complex - * when a real operator is applied to a complex right-hand side); the - * workspaces and the accumulation run in the promoted type. - * - * Evaluated as \c (B X) A^T, the first factor \c B X can overflow even when - * the product itself is representable (the multiplication by \c A^T rescales - * it: think \c B huge and \c A tiny). Each column is therefore pre-scaled by - * an exact power of two [2][3] derived from the component-wise exponent - * bounds of \c A, \c B and the column -- never from complex moduli, which - * overflow near the threshold -- so that no intermediate can spuriously - * overflow, and the - * exponent is folded back into the result through two representable - * half-factors, saturating to 0/Inf exactly when the true result leaves the - * representable range. The scale is one whenever the conservative bound shows - * no overflow risk, so results of moderate magnitude are bit-identical to the - * unscaled evaluation. Non-finite data is never scaled (the bounds cannot see - * past an Inf/NaN, and 0 * Inf would manufacture NaNs); the unscaled GEMMs - * propagate it entrywise exactly like a dense product. With a sparse factor, - * non-finite inputs instead use the stored-entry product to preserve the - * distinction between absent entries and stored zeros. */ + * \f$ (A \otimes B)\,\mathrm{vec}(X) = \mathrm{vec}(B X A^T) \f$ of [1], one + * product per factor -- a GEMM, an SpMM for a sparse factor, a scaling for a + * diagonal one, nothing for a square identity. A single right-hand side is + * reshaped in place, \f$ \mathrm{mat}(y) \mathrel{+}= \alpha B X A^T \f$ with + * \f$ X = \mathrm{mat}(x) \f$, so with a square identity factor the product is + * one pass of the other factor over \a x. Several right-hand sides are applied + * \c c at a time, see rhsChunk(): with \f$ X_k = \mathrm{mat}(x_k) \f$ stacked + * as block rows of \f$ \hat X \f$ (\c n2*c x \c n1), see stackColumns(), + * \f[ \hat Y = \big(B\,\hat X_{[n_2 \times c n_1]}\big)_{[m_2 c \times n_1]}\,A^T, + * \qquad \hat Y_k = B X_k A^T = \mathrm{mat}(y_k), \f] + * where \f$ M_{[p \times q]} \f$ is the column-major reshape. + * \c ProductScalar is the promoted scalar of the product (complex when a real + * operator is applied to a complex right-hand side); the workspaces and the + * accumulation run in the promoted type. As in any product chain, \f$ B X \f$ + * can overflow on extreme factor magnitudes (\c B huge, \c A tiny) even when + * the product is representable, and Inf/NaN propagate as through the two + * products. */ template <typename Dest, typename Rhs, typename ProductScalar> void addProduct(Dest& dst, const Rhs& rhs, const ProductScalar& alpha) const { - using ProductVector = Matrix<ProductScalar, Dynamic, 1>; using ProductMatrix = Matrix<ProductScalar, Dynamic, Dynamic, ColMajor>; // ColMajor: see DenseMatrix - using ProductReal = typename NumTraits<ProductScalar>::Real; - // In particular, materialize a nested matrix product once before taking - // per-column expressions from it. Otherwise each column evaluator may - // independently evaluate the entire nested product. + const Index m1 = m_A.rows(), n1 = m_A.cols(), m2 = m_B.rows(), n2 = m_B.cols(), r = rhs.cols(); + eigen_assert(rhs.rows() == n1 * n2 && "invalid product: dimensions do not match"); typename internal::nested_eval<Rhs, 1>::type actualRhs(rhs); - const Index n1 = m_A.cols(), n2 = m_B.cols(); - eigen_assert(actualRhs.rows() == n1 * n2 && "invalid product: dimensions do not match"); - // If max|X| < 2^eX, the two factor applications are bounded by - // 2^(eB+eX+bits2) and 2^(eA+eB+eX+bits1+bits2), including dot-product growth. - const bool factorsFinite = LhsOps::allFinite(m_A) && RhsOps::allFinite(m_B); - const int expA = LhsOps::exponentBound(m_A); - const int expB = RhsOps::exponentBound(m_B); - // Bit widths of the contracted dimensions, i.e. the dot-product growth - // bound; zero for a diagonal factor, whose application has no accumulation. - const int bits1 = LhsOps::growthBits(n1); - const int bits2 = RhsOps::growthBits(n2); - const int budget = std::numeric_limits<ProductReal>::max_exponent - 2; - ProductVector xc(n1 * n2); - ProductMatrix Y(m_B.rows(), m_A.rows()); - for (Index k = 0; k < actualRhs.cols(); ++k) { - xc = actualRhs.col(k).template cast<ProductScalar>(); - int e = 0; - const bool finite = factorsFinite && xc.allFinite(); - if (finite) { - const int expX = internal::structured_exponent_bound(xc); // 0 for an all-zero column: no scaling - e = numext::maxi(0, numext::maxi(expB + expX + bits2 - budget, expA + expB + expX + bits1 + bits2 - budget)); - // The cap keeps the half-factors below overflow; it only binds when the - // true result overflows anyway, so the saturation to Inf is genuine. - e = numext::mini(e, 2 * budget); + const bool skipA = LhsOps::isSquareIdentity(m_A), skipB = RhsOps::isSquareIdentity(m_B); + ProductMatrix X, BX, Y, work; + const Index chunk = rhsChunk<ProductScalar>(n1 * n2 + m2 * n1 + m2 * m1); + for (Index k0 = 0; k0 < r; k0 += chunk) { + const Index c = numext::mini(chunk, r - k0); + if (c == 1) { + const auto Xk = actualRhs.col(k0).reshaped(n2, n1); + auto Yk = dst.col(k0).reshaped(m2, m1); + if (skipA) { + RhsOps::addLeftProduct(Yk, alpha, m_B, Xk); + } else if (skipB) { + LhsOps::addRightProduct(Yk, alpha, Xk, m_A, work); + } else { + BX.setZero(m2, n1); + RhsOps::addLeftProduct(BX, ProductScalar(1), m_B, Xk); + LhsOps::addRightProduct(Yk, alpha, BX, m_A, work); + } + continue; } - if (e > 0) { - // Each power of two is split in halves so that the factors themselves - // stay representable; scaling by the two exact factors in sequence is - // still an exact shift wherever the result is representable. - const ProductReal down1 = ProductReal(std::ldexp(ProductReal(1), -(e / 2))); - const ProductReal down2 = ProductReal(std::ldexp(ProductReal(1), -(e - e / 2))); - xc = (xc * down1) * down2; + stackColumns(X, actualRhs.middleCols(k0, c), n2, n1); + if (!skipB) { + BX.setZero(m2, c * n1); + RhsOps::addLeftProduct(BX, ProductScalar(1), m_B, X.reshaped(n2, c * n1)); } - // For a diagonal factor its side degenerates to a diagonal scaling - // (transposedOperand: a diagonal matrix is its own transpose), a single - // stored term per entry: only a sparse factor can leave an exact 0 in B X - // (an empty row) for an Inf/NaN of A to meet. - if (EIGEN_PREDICT_FALSE(!finite && kHasSparseFactor)) - productNonFinite(Y, xc); - else - Y.noalias() = m_B * xc.reshaped(n2, n1) * LhsOps::transposedOperand(m_A); - if (e > 0) { - const ProductReal up1 = ProductReal(std::ldexp(ProductReal(1), e / 2)); - const ProductReal up2 = ProductReal(std::ldexp(ProductReal(1), e - e / 2)); - dst.col(k) += alpha * ((Y.reshaped() * up1) * up2); - } else { - dst.col(k) += alpha * Y.reshaped(); + const auto BXhat = (skipB ? X : BX).reshaped(m2 * c, n1); + if (!skipA) { + Y.setZero(m2 * c, m1); + LhsOps::addRightProduct(Y, ProductScalar(1), BXhat, m_A, work); + } + for (Index k = 0; k < c; ++k) { + auto Yk = dst.col(k0 + k).reshaped(m2, m1); + if (skipA) + Yk += alpha * BXhat.middleRows(k * m2, m2); + else + Yk += alpha * Y.middleRows(k * m2, m2); } } } private: - // A dense intermediate loses which zeros were structural; a later Inf/NaN - // factor would turn those absent terms into NaNs. Preserve the stored pairs. - template <typename ProductScalar> - EIGEN_DONT_INLINE void productNonFinite(Matrix<ProductScalar, Dynamic, Dynamic, ColMajor>& result, - const Matrix<ProductScalar, Dynamic, 1>& x) const { - result.setZero(); - LhsOps::forEachNonZero(m_A, [&result, &x, this](Index iA, Index jA, const Scalar& a) { - RhsOps::forEachNonZero(m_B, [&result, &x, jA, iA, &a, this](Index iB, Index jB, const Scalar& b) { - result.coeffRef(iB, iA) += (a * b) * x.coeff(jA * m_B.cols() + jB); - }); - }); + /** \internal Writes the columns of \a x, each reshaped to \a rows x \a cols, + * as the block rows of \a stacked, (\a rows * \c x.cols()) x \a cols. In this + * layout a factor applies to all columns with a single product: from the left + * to the \a rows x (\c x.cols() * \a cols) reshape, from the right to the + * matrix itself. Callers pass \a x through \c nested_eval first: a nested + * product would otherwise be evaluated anew for every column taken from it. */ + template <typename Stacked, typename Xpr> + static void stackColumns(Stacked& stacked, const Xpr& x, Index rows, Index cols) { + stacked.resize(rows * x.cols(), cols); + for (Index k = 0; k < x.cols(); ++k) + stacked.middleRows(k * rows, rows) = x.col(k).reshaped(rows, cols).template cast<typename Stacked::Scalar>(); } - /** \internal \returns the relative rank/pseudo-inversion threshold for the + /** \internal \returns how many right-hand sides to stack at once, given the + * \a perColumn workspace entries each one needs: as many as fit in an eighth + * of the L2 cache, at least one. Stacking widens the factor products, which + * pays for small factors; larger workspaces cost more in allocation and cache + * misses than they gain (measured in bench_structured_kronecker_batched). */ + template <typename WorkScalar> + static Index rhsChunk(Index perColumn) { + const Index budget = Index(l2CacheSize() / 8) / Index(sizeof(WorkScalar)); + return numext::maxi(Index(1), budget / numext::maxi(perColumn, Index(1))); + } + + /** \internal \returns the relative rank threshold for the * pairwise singular-value products, in the spirit of the SVD-based * pseudo-inverse: a mode \c (i,j) is kept when * \c (sa[i]/sa[0]) * (sb[j]/sb[0]) >= min(rows,cols) * epsilon, the \c SVDBase @@ -1068,8 +1341,7 @@ return RealScalar(numext::mini(rows(), cols())) * NumTraits<RealScalar>::epsilon(); } - // Factor-level ratios and frexp decompositions are independent of the RHS and - // of the other factor. Keep rank() and pseudo-inversion on the same predicate. + // Factor-level ratios and frexp decompositions, independent of the other factor. struct SingularModes { explicit SingularModes(const RealVector& s) : ratios(s.size()), mantissas(s.size()), exponents(s.size()) { for (Index i = 0; i < s.size(); ++i) { @@ -1081,7 +1353,7 @@ } bool retains(const SingularModes& other, Index i, Index j, RealScalar tol) const { - // Negation keeps NaN ratios in the inverted set, matching SVDBase. + // Negation counts NaN ratios as retained, matching SVDBase. if (ratios[i] * other.ratios[j] < tol) return false; const RealScalar m = mantissas[i] * other.mantissas[j]; if (!(numext::isfinite)(m)) return true; @@ -1096,6 +1368,34 @@ Matrix<int, Dynamic, 1> exponents; }; + /** \internal \returns the mantissa of \c det(*this) in the balanced form + * \c m * 2^e, adding \c e into \a exponent; see determinant(). */ + Scalar balancedDeterminant(Index& exponent) const { + eigen_assert(m_A.rows() == m_A.cols() && m_B.rows() == m_B.cols() && + "KroneckerOperator::determinant requires square factors"); + const Scalar mant = balancedDetPow(m_A, m_B.cols(), exponent); + return internal::structured_balance(mant * balancedDetPow(m_B, m_A.cols(), exponent), exponent); + } + + /** \internal Writes \f$ (A \otimes B)^{-1} b \f$ into \a x through the factor + * solvers \a solverA and \a solverB of the square factors of sizes \a n1 and + * \a n2: \f$ X = B^{-1} \mathrm{mat}(b) A^{-T} \f$, the right-hand sides + * stacked as in addProduct(). */ + template <typename SolverA, typename SolverB, typename Rhs, typename Dest> + static void solveWith(const SolverA& solverA, const SolverB& solverB, Index n1, Index n2, const Rhs& b, Dest& x) { + typename internal::nested_eval<Rhs, 1>::type actualRhs(b); + DenseMatrix S, Z, X; + const Index r = b.cols(); + const Index chunk = rhsChunk<Scalar>(3 * n1 * n2); + for (Index k0 = 0; k0 < r; k0 += chunk) { + const Index c = numext::mini(chunk, r - k0); + stackColumns(S, actualRhs.middleCols(k0, c), n2, n1); + Z = solverB.solveLeft(S.reshaped(n2, c * n1)); + X = solverA.solveTransposedRight(Z.reshaped(n2 * c, n1)); + for (Index k = 0; k < c; ++k) x.col(k0 + k).reshaped(n2, n1) = X.middleRows(k * n2, n2); + } + } + /** \internal \returns the mantissa of \c det(M)^power in the balanced form * \c m * 2^e, adding \c e into \a exponent. The determinant is accumulated * directly from the LU diagonal, times the permutation sign (from the @@ -1122,17 +1422,28 @@ /** \ingroup StructuredMatrices_Module * \returns a \ref KroneckerOperator \c a (x) \c b holding evaluated copies of the - * factors. The operator type is deduced from the plain types of \a a and \a b: - * a dense argument becomes a \c Matrix factor, a diagonal one (e.g. - * \c VectorXd::Ones(n).asDiagonal() for an identity factor) an owning - * \c DiagonalMatrix, a sparse one a compressed \c SparseMatrix, each exploited - * structurally, see \ref KroneckerOperator. */ + * factors. The operator type is deduced from the types of \a a and \a b: a + * dense argument becomes a \c Matrix factor, a diagonal one an owning + * \c DiagonalMatrix, a sparse one a compressed \c SparseMatrix, an + * \c Identity() expression an identity factor holding its dimensions, and a + * \c KroneckerOperator is stored as it is, each exploited structurally, see + * \ref KroneckerOperator. */ template <typename LhsDerived, typename RhsDerived> -KroneckerOperator<typename LhsDerived::PlainObject, typename RhsDerived::PlainObject> makeKroneckerOperator( - const EigenBase<LhsDerived>& a, const EigenBase<RhsDerived>& b) { +KroneckerOperator<typename internal::kron_factor_storage<LhsDerived>::type, + typename internal::kron_factor_storage<RhsDerived>::type> +makeKroneckerOperator(const EigenBase<LhsDerived>& a, const EigenBase<RhsDerived>& b) { return {a.derived(), b.derived()}; } +/** \ingroup StructuredMatrices_Module + * \returns the \ref KroneckerOperator \c a (x) \c b (x) \c c (x) ..., nested to + * the right: \c makeKroneckerOperator(a, makeKroneckerOperator(b, c, ...)). */ +template <typename D1, typename D2, typename D3, typename... Rest> +auto makeKroneckerOperator(const EigenBase<D1>& a, const EigenBase<D2>& b, const EigenBase<D3>& c, + const Rest&... rest) { + return makeKroneckerOperator(a, makeKroneckerOperator(b, c, rest...)); +} + namespace internal { template <typename LhsMatrix, typename RhsMatrix, typename Rhs, int ProductTag>
diff --git a/contrib/Eigen/src/StructuredMatrices/StructuredMatrixUtils.h b/contrib/Eigen/src/StructuredMatrices/StructuredMatrixUtils.h index ab4cbd9..917dc30 100644 --- a/contrib/Eigen/src/StructuredMatrices/StructuredMatrixUtils.h +++ b/contrib/Eigen/src/StructuredMatrices/StructuredMatrixUtils.h
@@ -11,10 +11,10 @@ // [1] N. J. Higham, "Accuracy and Stability of Numerical Algorithms", 2nd ed., // SIAM, 2002, chapter 27. Avoiding spurious overflow by rescaling with // powers of two, the technique behind structured_exponent_bound() and the -// column scaling in structured_fft_apply() and Bccb::applySymbol(). +// balanced determinant accumulations. // [2] P. H. Sterbenz, "Floating-Point Computation", Prentice-Hall, 1974. -// Scaling by a power of two is exact, so the scaled transforms introduce -// no roundoff beyond the transforms themselves. +// Scaling by a power of two is exact, so the balanced accumulations +// introduce no roundoff of their own. #ifndef EIGEN_STRUCTURED_MATRIX_UTILS_H #define EIGEN_STRUCTURED_MATRIX_UTILS_H @@ -44,6 +44,29 @@ // exponents for dimensions that are otherwise practical. using structured_exponent_type = numext::int64_t; +/** \internal max(|re z|, |im z|), the component magnitude the balanced forms + * key on; from the representation for std::complex<float> and + * std::complex<double>, where a flush-to-zero comparison would read a subnormal + * component as zero. */ +template <typename Scalar> +typename NumTraits<Scalar>::Real structured_component_magnitude_impl(const Scalar& z, std::true_type) { + using RealScalar = typename NumTraits<Scalar>::Real; + using Binary = binary_floating_point_traits<RealScalar>; + return numext::bit_cast<RealScalar>( + larger_magnitude_bits<RealScalar>(Binary::magnitude(numext::real(z)), Binary::magnitude(numext::imag(z)))); +} +template <typename Scalar> +typename NumTraits<Scalar>::Real structured_component_magnitude_impl(const Scalar& z, std::false_type) { + return numext::maxi(numext::abs(numext::real(z)), numext::abs(numext::imag(z))); +} +template <typename Scalar> +typename NumTraits<Scalar>::Real structured_component_magnitude(const Scalar& z) { + using RealScalar = typename NumTraits<Scalar>::Real; + return structured_component_magnitude_impl(z, + bool_constant < complex_array_access<Scalar>::value && + use_subnormal_preserving_scaling<RealScalar, RealScalar>::value > ()); +} + /** \internal Balanced mantissa*2^e arithmetic shared by the structured * operators' determinant-style accumulations (the split fraction/exponent * convention of LINPACK's xGEDI; see the per-class references). @@ -52,22 +75,25 @@ * or \c |z| for a real scalar -- into [0.5, 1), accumulating the removed * exponent into \a exponent. The rescaling is exact, so no roundoff is * introduced; zeros and non-finite values, which must propagate exactly, are - * returned untouched. */ + * returned untouched. The tests and the scalings read the representation for + * float and double (numext::is_exactly_zero_no_flush(), + * internal::frexp_exponent_preserving_subnormals(), + * internal::ldexp_preserving_subnormals()): under flush-to-zero a comparison + * or a frexp on a subnormal recovered from a flushed reduction would read it as + * zero again. */ template <typename Scalar, bool IsComplex = NumTraits<Scalar>::IsComplex> struct structured_balance_impl { using RealScalar = typename NumTraits<Scalar>::Real; template <typename Exponent> static Scalar run(const Scalar& z, Exponent& exponent) { - const RealScalar mag = numext::maxi(numext::abs(numext::real(z)), numext::abs(numext::imag(z))); - if (!(mag > RealScalar(0)) || !(numext::isfinite)(mag)) return z; - int e; - EIGEN_USING_STD(frexp); - frexp(mag, &e); + const RealScalar mag = structured_component_magnitude(z); + if (numext::is_exactly_zero_no_flush(mag) || !(numext::isfinite)(mag)) return z; + const int e = frexp_exponent_preserving_subnormals(mag); exponent += e; - return Scalar(numext::ldexp(numext::real(z), -e), numext::ldexp(numext::imag(z), -e)); + return apply_exponent(z, -e); } static Scalar apply_exponent(const Scalar& z, int e) { - return Scalar(numext::ldexp(numext::real(z), e), numext::ldexp(numext::imag(z), e)); + return Scalar(ldexp_preserving_subnormals(numext::real(z), e), ldexp_preserving_subnormals(numext::imag(z), e)); } }; @@ -75,15 +101,12 @@ struct structured_balance_impl<Scalar, false> { template <typename Exponent> static Scalar run(const Scalar& x, Exponent& exponent) { - const Scalar mag = numext::abs(x); - if (!(mag > Scalar(0)) || !(numext::isfinite)(mag)) return x; - int e; - EIGEN_USING_STD(frexp); - frexp(mag, &e); + if (numext::is_exactly_zero_no_flush(x) || !(numext::isfinite)(x)) return x; + const int e = frexp_exponent_preserving_subnormals(x); exponent += e; - return numext::ldexp(x, -e); + return apply_exponent(x, -e); } - static Scalar apply_exponent(const Scalar& x, int e) { return numext::ldexp(x, e); } + static Scalar apply_exponent(const Scalar& x, int e) { return ldexp_preserving_subnormals(x, e); } }; template <typename Scalar, typename Exponent> @@ -102,127 +125,6 @@ return structured_balance_impl<Scalar>::apply_exponent(z, e); } -/** \internal \returns 1 / \a z, formed in a balanced frame where the direct - * reciprocal is unreliable: near the representable boundary both conj(z)/|z|^2 - * and Smith's algorithm (internal::complex_divide_fast and _smith) overflow to - * zero when |Re z| and |Im z| are each around half the largest finite value. - * Balancing puts the larger component in [0.5, 1), where the reciprocal is - * always representable, and both rescalings are exact. Zeros and non-finite - * values pass through structured_balance() untouched. */ -template <typename Scalar> -Scalar structured_scaled_reciprocal(const Scalar& z) { - using RealScalar = typename NumTraits<Scalar>::Real; - const RealScalar mag = numext::maxi(numext::abs(numext::real(z)), numext::abs(numext::imag(z))); - const RealScalar m2 = mag * mag; - // Where |z|^2 <= 2*mag^2 is normal, no implementation of the reciprocal can - // overflow or underflow, so only the remaining entries pay for the balancing. - if (m2 >= (std::numeric_limits<RealScalar>::min)() && (numext::isfinite)(m2 + m2)) return Scalar(1) / z; - Index e = 0; - const Scalar zs = structured_balance(z, e); - return structured_ldexp_clamped(Scalar(1) / zs, -e); -} - -template <typename RealScalar> -struct structured_reciprocal_packet_traits { - // These backends provide interleaved complex packets, ploaddup and mask reductions. -#if defined(EIGEN_VECTORIZE_SSE2) - static constexpr bool Vectorizable = - std::is_same<RealScalar, float>::value || std::is_same<RealScalar, double>::value; -#else - static constexpr bool Vectorizable = false; -#endif -}; - -template <typename RealScalar, bool ApplyThreshold, - bool Vectorize = structured_reciprocal_packet_traits<RealScalar>::Vectorizable> -struct structured_symbol_reciprocal_impl { - using Complex = std::complex<RealScalar>; - static void run(const Complex* symbol, const RealScalar* moduli, RealScalar tol, Complex* inverse, Index size) { - for (Index k = 0; k < size; ++k) - inverse[k] = ApplyThreshold && moduli[k] < tol ? Complex(0) : structured_scaled_reciprocal(symbol[k]); - } -}; - -template <typename RealScalar, bool ApplyThreshold> -struct structured_symbol_reciprocal_impl<RealScalar, ApplyThreshold, true> { - using Complex = std::complex<RealScalar>; - using Packet = typename packet_traits<Complex>::type; - using RealPacket = typename unpacket_traits<Packet>::as_real; - - EIGEN_DONT_INLINE static void run(const Complex* symbol, const RealScalar* moduli, RealScalar tol, Complex* inverse, - Index size) { - constexpr Index kPacketSize = unpacket_traits<Packet>::size; - const Packet zero = pset1<Packet>(Complex(0)); - const Packet safe_value = pset1<Packet>(Complex(1, 1)); - const RealPacket threshold = pset1<RealPacket>(tol); - // min_normal <= mag^2 and 2*mag^2 <= 2^(max_exponent-1) < max_finite. - const RealPacket lower = - pset1<RealPacket>(numext::ldexp(RealScalar(1), std::numeric_limits<RealScalar>::min_exponent / 2)); - const RealPacket upper = - pset1<RealPacket>(numext::ldexp(RealScalar(1), std::numeric_limits<RealScalar>::max_exponent / 2 - 1)); - const Index packet_end = size - size % kPacketSize; - Index k = 0; - for (; k < packet_end; k += kPacketSize) { - const RealPacket discard = ApplyThreshold ? pcmp_lt(ploaddup<RealPacket>(moduli + k), threshold) : zero.v; - // Sanitize before arithmetic: selecting the output alone still divides discarded zeros. - const Packet z = pselect(Packet(discard), safe_value, ploadu<Packet>(symbol + k)); - const RealPacket a = pabs(z.v); - const RealPacket mag = pmax(a, pcplxflip(Packet(a)).v); - // Zero components use the scalar path to preserve std::complex's signed-zero choices. - const RealPacket safe = pand(pand(pcmp_le(lower, mag), pcmp_le(mag, upper)), pcmp_lt(zero.v, a)); - if (!predux_any(pandnot(ptrue(safe), safe))) { - const RealPacket square = pmul(z.v, z.v); - const RealPacket denom = padd(square, pcplxflip(Packet(square)).v); - const Packet reciprocal(pdiv(pconj(z).v, denom)); - // Underflowed components also retain the scalar signed-zero convention. - if (!predux_any(pcmp_eq(reciprocal.v, zero.v))) { - pstoreu(inverse + k, pselect(Packet(discard), zero, reciprocal)); - continue; - } - } - structured_symbol_reciprocal_impl<RealScalar, ApplyThreshold, false>::run( - symbol + k, ApplyThreshold ? moduli + k : nullptr, tol, inverse + k, kPacketSize); - } - if (k < size) - structured_symbol_reciprocal_impl<RealScalar, ApplyThreshold, false>::run( - symbol + k, ApplyThreshold ? moduli + k : nullptr, tol, inverse + k, size - k); - } -}; - -// The input and output may alias exactly. Threshold equality and NaN moduli are retained. -template <typename RealScalar> -void structured_symbol_reciprocal(const std::complex<RealScalar>* symbol, const RealScalar* moduli, RealScalar tol, - std::complex<RealScalar>* inverse, Index size) { - structured_symbol_reciprocal_impl<RealScalar, true>::run(symbol, moduli, tol, inverse, size); -} - -// True inverses retain every mode and reuse the symbol storage without allocating a modulus array. -template <typename RealScalar> -void structured_symbol_reciprocal(std::complex<RealScalar>* symbol, Index size) { - structured_symbol_reciprocal_impl<RealScalar, false>::run(symbol, nullptr, RealScalar(0), symbol, size); -} - -/** \internal \returns \c a - b guarded against spurious overflow: when the - * plain difference of two finite values overflows to infinity, it is - * recomputed from the halved operands with \a e set to 1 so the caller can - * carry the factor of two in its running exponent (the balanced accumulations - * fold it into \c exponent at the call site). The recomputation is exact where - * it matters: a difference of finite values only overflows when both operands - * are huge, normal values, whose halves and halved difference are exactly - * representable. For complex scalars the finiteness tests are component-wise; - * a subnormal component riding along with a huge one loses its last bit to the - * halving, an error far below one ulp of the factor's magnitude. Non-finite - * operands (genuine Inf/NaN nodes) propagate untouched, with \a e = 0. */ -template <typename Scalar> -Scalar structured_guarded_diff(const Scalar& a, const Scalar& b, int& e) { - using RealScalar = typename NumTraits<Scalar>::Real; - e = 0; - const Scalar t = a - b; - if ((numext::isfinite)(t) || !(numext::isfinite)(a) || !(numext::isfinite)(b)) return t; - e = 1; - return a * RealScalar(0.5) - b * RealScalar(0.5); -} - /** \internal \returns the indices sorted by decreasing precomputed modulus * \a mods (each modulus is computed once, not on every comparison); the shared * ordering of the operators' singularValues()/matrixU()/matrixV(). The sort is @@ -345,11 +247,14 @@ m = x.realView().cwiseAbs().maxCoeff(); else m = x.cwiseAbs().maxCoeff(); + // A SIMD unit that flushes subnormal inputs (ARMv7 NEON, Arm FZ, DAZ) reads + // an all-subnormal operand as zero; the rescan recovers the largest component + // from its representation. + m = safe_scaling<RealScalar>::recover_flushed_max_coeff(x, m); e = 0; if (!(numext::isfinite)(m)) return false; - if (m > RealScalar(0)) { - EIGEN_USING_STD(frexp); - frexp(m, &e); + if (!numext::is_exactly_zero_no_flush(m)) { + e = frexp_exponent_preserving_subnormals(m); if (ScalarTraits::IsComplex) ++e; } return true; @@ -364,6 +269,100 @@ return e; } +// The packet form of M *= 2^e. ldexp saturates entrywise for every e without +// forming a possibly unrepresentable 2^e; for std::complex storage, realView() +// exposes both components to the real ldexp packets. +template <typename Xpr, std::enable_if_t<!NumTraits<typename Xpr::Scalar>::IsComplex, bool> = true> +void structured_ldexp_entries_packet(Xpr& M, int e) { + M.array() = M.array().ldexp(e); +} + +template <typename Xpr, std::enable_if_t<complex_array_access<typename Xpr::Scalar>::value, bool> = true> +void structured_ldexp_entries_packet(Xpr& M, int e) { + M.realView().array() = M.realView().array().ldexp(e); +} + +template <typename Xpr, std::enable_if_t<NumTraits<typename Xpr::Scalar>::IsComplex && + !complex_array_access<typename Xpr::Scalar>::value, + bool> = true> +void structured_ldexp_entries_packet(Xpr& M, int e) { + using Scalar = typename Xpr::Scalar; + M = M.unaryExpr([e](const Scalar& z) { return structured_ldexp_clamped(z, Index(e)); }); +} + +template <typename Xpr> +void structured_ldexp_entries_impl(Xpr& M, int e, int, std::false_type) { + structured_ldexp_entries_packet(M, e); +} + +template <typename Xpr> +void structured_ldexp_entries_impl(Xpr& M, int e, int bound, std::true_type) { + using Scalar = typename Xpr::Scalar; + using RealScalar = typename NumTraits<Scalar>::Real; + // The largest component is at least 2^(componentBound - 1): a complex bound + // carries one extra bit for the modulus. The input holds no significant + // subnormal when componentBound - 1 >= recovery, and neither does the result + // when componentBound - 1 + e >= recovery. + const int componentBound = bound - (NumTraits<Scalar>::IsComplex ? 1 : 0); + const int recovery = safe_scaling<RealScalar>::subnormal_recovery_exponent(); + if (componentBound - 1 >= recovery && componentBound - 1 + e >= recovery) + structured_ldexp_entries_packet(M, e); + else + M = M.unaryExpr(scale_by_exponent_op<RealScalar>(e)); +} + +/** \internal M *= 2^e, exactly wherever the result is representable, with + * ldexp's saturation beyond. \a bound is the exponent bound of the input, + * max|M| < 2^bound with 2^(bound - 1) <= max|M| as structured_exponent_bound() + * returns it, or 0 for zero and non-finite data. Coefficients within a factor + * 2^(1 - digits) of the largest one are exact: where the input or the result + * can hold subnormals among them the scaling goes through integer significands + * (internal::scale_by_exponent_op), which FTZ/DAZ hardware and ARMv7 NEON + * cannot flush; elsewhere the packet ldexp applies, and coefficients further + * below the largest may still flush under FTZ/DAZ. See + * structured_ldexp_entries_exact() for data where every coefficient matters. */ +template <typename Xpr> +void structured_ldexp_entries(Xpr& M, int e, int bound) { + if (e == 0) return; + using Scalar = typename Xpr::Scalar; + using RealScalar = typename NumTraits<Scalar>::Real; + structured_ldexp_entries_impl(M, e, bound, + bool_constant<use_subnormal_preserving_scaling<RealScalar, Scalar>::value>()); +} + +/** \internal M *= 2^e with the exponent bound taken from M itself, for data + * whose bound the caller does not track, such as a result being folded back + * from a normalized frame. */ +template <typename Xpr> +void structured_ldexp_entries(Xpr& M, int e) { + if (e == 0) return; + structured_ldexp_entries(M, e, structured_exponent_bound(M)); +} + +template <typename Xpr> +void structured_ldexp_entries_exact_impl(Xpr& M, int e, std::false_type) { + structured_ldexp_entries_packet(M, e); +} +template <typename Xpr> +void structured_ldexp_entries_exact_impl(Xpr& M, int e, std::true_type) { + using RealScalar = typename NumTraits<typename Xpr::Scalar>::Real; + M = M.unaryExpr(scale_by_exponent_op<RealScalar>(e)); +} + +/** \internal M *= 2^e through integer significands for float and double + * whatever the magnitudes: for the factors of an LU, the poles of a secular + * equation or a spectrum, a coefficient far below the largest one still + * matters, and the packet path may flush it under FTZ/DAZ. The pass is scalar; + * use it where it precedes a factorization or covers O(n) data. */ +template <typename Xpr> +void structured_ldexp_entries_exact(Xpr& M, int e) { + if (e == 0) return; + using Scalar = typename Xpr::Scalar; + using RealScalar = typename NumTraits<Scalar>::Real; + structured_ldexp_entries_exact_impl(M, e, + bool_constant<use_subnormal_preserving_scaling<RealScalar, Scalar>::value>()); +} + /** \internal \returns the index reversal of a DFT \a symbol: result[k] = * symbol[(p - k) mod p]. This is the symbol of the transposed operator: reversing * the generating sequence in index space reverses the frequencies of its DFT, for @@ -380,163 +379,70 @@ return reversed; } -/** \internal Default pointwise step: multiply the transformed column by the symbol. */ -struct structured_symbol_multiply { - template <typename SymbolType> - int exponent_growth(const SymbolType& symbol) const { - return structured_exponent_bound(symbol); - } - template <typename XfType, typename SymbolType> - void operator()(XfType& xf, const SymbolType& symbol) const { - xf.array() *= symbol.array(); - } -}; - -/** \internal - * By default computes - * \c dst.col(k) += alpha * ifft( symbol .* fft(rhs.col(k)) ) for every column of - * \a rhs, i.e. applies the circulant operator whose eigenvalues are \a symbol. - * A caller may supply another pointwise operation with its matching exponent - * growth bound. The leading \a outSize entries of each back-transform form the - * corresponding output column. All workspace is allocated once outside the - * per-column loop; right-hand sides shorter than the transform length are - * zero-padded into the preallocated buffer so the FFT never re-allocates. +/** \internal Computes \c dst.col(k) += alpha * ifft( symbol .* fft(rhs.col(k)) ) for + * every column of \a rhs, i.e. applies the circulant operator whose eigenvalues + * are \a symbol; the leading \a outSize entries of each back-transform form the + * output column. Right-hand sides shorter than the transform length are + * zero-padded into a buffer allocated once outside the column loop. * - * The transforms accumulate up to \c p addends, so a finite column near the - * overflow threshold (or one applied through a symbol of huge magnitude) would - * overflow inside the FFT and turn a representable result into Inf/NaN. Each - * column is therefore scaled down by a power of two -- an exact shift, no - * roundoff -- derived from the column's exponent and the pointwise operation's - * maximum exponent growth, and the exponent is folded back into the output after - * the back-transform. The scale is one whenever the conservative intermediate - * bound cannot overflow, so results are bit-identical for inputs of moderate - * magnitude; zero columns are never scaled. If \f$|x_i|<2^c\f$ and the pointwise - * operation magnifies by less than \f$2^g\f$, its result and both transforms are - * bounded by - * \f[ 2^{c+g+2\lceil\log_2 p\rceil+1}. \f] - * - * Genuinely non-finite data must not go through the transforms: they mix every - * input entry into every output entry, so a single special value would - * contaminate the whole column with NaN where the dense product only propagates - * it through the dot products that touch it. A non-finite symbol is the - * caller's responsibility (the operators route it to their direct kernels up - * front); a non-finite column is detected here -- in the same single pass that - * derives its scaling exponent -- and handed to \a directColumn, the caller's - * per-column direct kernel, so the remaining columns keep the fast path. - */ -template <typename Scalar, typename Dest, typename Rhs, typename DirectColumn, - typename Pointwise = structured_symbol_multiply> + * The transforms are evaluated in plain floating-point arithmetic: a column within + * a factor of about p * max|symbol| of the overflow threshold can overflow in the + * intermediates even when the result is representable, and a single Inf or NaN + * makes the whole output column non-finite. */ +template <typename Scalar, typename Dest, typename Rhs> void structured_fft_apply(Dest& dst, const Matrix<std::complex<typename NumTraits<Scalar>::Real>, Dynamic, 1>& symbol, - Index outSize, const Rhs& rhs, const Scalar& alpha, DirectColumn&& directColumn, - Pointwise pointwise = Pointwise(), Index outExpAdjust = 0) { + Index outSize, const Rhs& rhs, const Scalar& alpha) { using RealScalar = typename NumTraits<Scalar>::Real; using Complex = std::complex<RealScalar>; using ComplexVector = Matrix<Complex, Dynamic, 1>; const Index p = symbol.size(); eigen_assert(rhs.rows() <= p && outSize <= p); - - // Growth of the pointwise step -- max|symbol| for a multiply, the largest - // retained reciprocal for a divide -- included in the pre-scaling so a - // representable result cannot overflow before the output rescaling. - const int pointwiseExp = pointwise.exponent_growth(symbol); - if (p == 1) { - // The length-one DFT is the identity and is unsupported by kissfft. The - // pointwise step still runs, scaled around just as the transforms are on the - // general path. - const int budget = NumTraits<RealScalar>::max_exponent() - 2; - ComplexVector xf(1); - for (Index k = 0; k < rhs.cols(); ++k) { - int colExp; - if (!structured_exponent_bound_finite(rhs.col(k), colExp)) { - directColumn(k); - continue; - } - const int e = numext::maxi(colExp + pointwiseExp - budget, 0); - const RealScalar down1 = numext::ldexp(RealScalar(1), -(e / 2)), - down2 = numext::ldexp(RealScalar(1), -(e - e / 2)); - const int eo = e - static_cast<int>(outExpAdjust); - const RealScalar up1 = numext::ldexp(RealScalar(1), eo / 2), up2 = numext::ldexp(RealScalar(1), eo - eo / 2); - xf.coeffRef(0) = static_cast<Complex>((rhs.coeff(0, k) * down1) * down2); - pointwise(xf, symbol); - dst.coeffRef(0, k) += alpha * structured_scalar_part_impl<Scalar>::run_scalar((xf.coeff(0) * up1) * up2); - } + // The length-one DFT is the identity and is unsupported by kissfft. + dst.row(0) += alpha * structured_scalar_part_impl<Scalar>::run(Complex(symbol.coeff(0)) * + rhs.row(0).template cast<Complex>()); return; } - // The bit width of p (>= 1 here), an upper bound for the ceil(log2 p) of the - // magnitude bound above -- one bit looser at power-of-two transform lengths. - const int log2p = log2_floor(static_cast<std::make_unsigned_t<Index>>(p)) + 1; - const int budget = NumTraits<RealScalar>::max_exponent() - 2 * log2p - 2; - auto&& fft = structured_fft_engine<RealScalar>(); ComplexVector xt = ComplexVector::Zero(p); ComplexVector xf(p), yt(p); for (Index k = 0; k < rhs.cols(); ++k) { - int colExp; // 0 for an all-zero column: no scaling - if (!structured_exponent_bound_finite(rhs.col(k), colExp)) { - directColumn(k); - continue; - } - const int e = numext::maxi(colExp + pointwiseExp - budget, 0); - // Each power of two is split in halves so that the factors themselves stay - // inside the exponent range even when e exceeds it (a huge column applied - // through a huge symbol); scaling by the two exact factors in sequence is - // still an exact shift wherever the result is representable. - const RealScalar down1 = numext::ldexp(RealScalar(1), -(e / 2)), down2 = numext::ldexp(RealScalar(1), -(e - e / 2)); - // A caller that pre-balanced the symbol passes its exponent in outExpAdjust, - // so the pointwise step sees an O(1) symbol whose result never passes through - // a subnormal; the exponent is folded back into the output here. - const int eo = e - static_cast<int>(outExpAdjust); - const RealScalar up1 = numext::ldexp(RealScalar(1), eo / 2), up2 = numext::ldexp(RealScalar(1), eo - eo / 2); - xt.head(rhs.rows()) = ((rhs.col(k) * down1) * down2).template cast<Complex>(); + xt.head(rhs.rows()) = rhs.col(k).template cast<Complex>(); fft.fwd(xf, xt, p); - pointwise(xf, symbol); + xf.array() *= symbol.array(); fft.inv(yt, xf, p); - dst.col(k) += alpha * structured_scalar_part_impl<Scalar>::run((yt.head(outSize) * up1) * up2); + dst.col(k) += alpha * structured_scalar_part_impl<Scalar>::run(yt.head(outSize)); } } -/** \internal Overload for callers that guarantee finite right-hand-side data - * (e.g. solve(), which checks its input up front and takes a dedicated - * pseudo-inverse fallback otherwise): a non-finite column has no direct kernel - * here and would be skipped, so it asserts. */ -template <typename Scalar, typename Dest, typename Rhs> -void structured_fft_apply(Dest& dst, const Matrix<std::complex<typename NumTraits<Scalar>::Real>, Dynamic, 1>& symbol, - Index outSize, const Rhs& rhs, const Scalar& alpha) { - structured_fft_apply(dst, symbol, outSize, rhs, alpha, - [](Index) { eigen_assert(false && "non-finite column requires a direct kernel"); }); +/** \internal \returns the pseudo-inverse symbol of a circulant-diagonalized + * operator: the reciprocal of every entry of \a symbol whose modulus is at least + * \c tol, zero for the others. With \c w = 1/|z| the reciprocal is formed as + * \c (conj(z) w) w, which stays representable wherever 1/z is and vectorizes as + * plain products. A NaN entry fails the comparison and stays in the inverted set, + * so it propagates instead of being silently truncated. */ +template <typename SymbolType, typename ModsType, typename RealScalar> +SymbolType structured_pinv_symbol(const SymbolType& symbol, const ModsType& mods, const RealScalar& tol) { + using Complex = typename SymbolType::Scalar; + const auto w = (mods.array() < tol).select(RealScalar(0), mods.array().inverse()).template cast<Complex>().eval(); + return (symbol.array().conjugate() * w * w).matrix(); } -/** \internal Pointwise step for the pseudo-inverse solve: divides by the symbol - * instead of multiplying by its reciprocal, since the reciprocal of an entry near - * the representable boundary is subnormal -- and lost under flush-to-zero -- while - * the quotient stays normal. Entries below the rank threshold are zeroed, matching - * the pseudo-inverse convention. */ -template <typename ModsType, typename RealScalar> -struct structured_symbol_divide { - const ModsType* mods; - RealScalar tol; - template <typename SymbolType> - int exponent_growth(const SymbolType&) const { - if (mods->size() == 0) return 0; - const auto retained = (RealScalar(1) - mods->cwiseTypedLess(tol).array()) * - mods->cwiseTypedGreater(RealScalar(0)).array() * mods->array().isFiniteTyped(); - const RealScalar smallest = retained.select(mods->array(), NumTraits<RealScalar>::infinity()).minCoeff(); - if (!(numext::isfinite)(smallest)) return 0; - int modExp; - EIGEN_USING_STD(frexp); - frexp(smallest, &modExp); - // The smallest retained modulus maximizes 2-frexp(mod).exponent. - return numext::maxi(0, 2 - modExp); - } - template <typename XfType, typename SymbolType> - void operator()(XfType& xf, const SymbolType& symbol) const { - using Complex = std::complex<RealScalar>; - xf.array() = (mods->array() < tol).select(Complex(0), xf.array() / symbol.array()); - } -}; +/** \internal \returns the rank threshold \c size * epsilon * max|symbol| of [Golub + * and Van Loan, Matrix Computations, 4th ed., 5.4], clamped from below by the + * smallest normal number like SVDBase::rank(); \a mods holds |symbol|. A finite + * entry whose modulus overflows would make the threshold infinite and truncate + * every mode, so that rare case takes the maximum from the halved symbol. */ +template <typename SymbolType, typename ModsType> +typename ModsType::Scalar structured_rank_threshold(const SymbolType& symbol, const ModsType& mods) { + using RealScalar = typename ModsType::Scalar; + const RealScalar factor = RealScalar(mods.size()) * NumTraits<RealScalar>::epsilon(); + RealScalar tol = factor * mods.maxCoeff(); + if (!(numext::isfinite)(tol)) tol = (RealScalar(2) * factor) * (symbol * RealScalar(0.5)).cwiseAbs().maxCoeff(); + return numext::maxi(tol, (std::numeric_limits<RealScalar>::min)()); +} /** \internal Shared product implementation for the structured operator types. * Forwards to the operator's \c addProduct member, which performs the fast
diff --git a/contrib/Eigen/src/StructuredMatrices/Toeplitz.h b/contrib/Eigen/src/StructuredMatrices/Toeplitz.h index a166c2b..01456c8 100644 --- a/contrib/Eigen/src/StructuredMatrices/Toeplitz.h +++ b/contrib/Eigen/src/StructuredMatrices/Toeplitz.h
@@ -117,7 +117,6 @@ // no consumer, so skip the embedding altogether. if (rows() > internal::structured_direct_threshold() || cols() > internal::structured_direct_threshold()) m_symbol = computeSymbol(); - m_fftUsable = computeFftUsable(); } EIGEN_DEVICE_FUNC Index rows() const { return m_col.size(); } @@ -249,26 +248,16 @@ /** \internal Computes \c dst += alpha * (*this) * rhs. \c ProductScalar is the * promoted scalar of the product (complex when a real operator is applied to a - * complex right-hand side); the accumulation runs in the promoted type. - * - * Non-finite data takes the direct O(mn) kernel: the transforms would smear a - * single Inf/NaN into NaNs across the whole output, where the dense product - * only propagates it through the dot products that touch it. Non-finite - * generators or a non-finite cached embedding symbol (which can overflow even - * for finite generators) route the whole product; a non-finite right-hand-side - * column is detected inside the FFT loop -- in the same pass that derives its - * scaling exponent, so finite data pays no extra scan -- and falls back per - * column. */ + * complex right-hand side); the accumulation runs in the promoted type. */ template <typename Dest, typename Rhs, typename ProductScalar> void addProduct(Dest& dst, const Rhs& rhs, const ProductScalar& alpha) const { const Index m = rows(), n = cols(); eigen_assert(rhs.rows() == n && "invalid product: dimensions do not match"); const bool small = m <= internal::structured_direct_threshold() && n <= internal::structured_direct_threshold(); - if (small || !m_fftUsable) + if (small) directProduct(dst, rhs, alpha); else - internal::structured_fft_apply(dst, m_symbol, m, rhs, alpha, - [&](Index k) { directProductColumn(dst, rhs, k, alpha); }); + internal::structured_fft_apply(dst, m_symbol, m, rhs, alpha); } private: @@ -282,23 +271,11 @@ * by transpose(), conjugate() and adjoint(), whose symbols are cheap * transformations of the existing one. */ Toeplitz(const ColGeneratorType& col, const RowGeneratorType& row, const ComplexVector& symbol) - : m_col(col), m_row(row), m_symbol(symbol) { - m_fftUsable = computeFftUsable(); - } - - /** \internal Whether products may take the FFT path: the generators and the - * cached embedding symbol must be finite. The symbol accumulates up to p - * addends, so it can overflow to Inf even for finite generators; such operators - * fall back to the direct kernel, which stays exact. */ - bool computeFftUsable() const { - return m_col.allFinite() && m_row.allFinite() && (m_symbol.size() == 0 || m_symbol.allFinite()); - } + : m_col(col), m_row(row), m_symbol(symbol) {} /** \internal Direct O(mn) kernel for column \a k of the right-hand side: - * computes \c dst.col(k) += alpha * (*this) * rhs.col(k) without transforms. - * Serves operators below the FFT threshold and any column involving - * non-finite data, whose entrywise IEEE semantics the transforms cannot - * preserve. */ + * computes \c dst.col(k) += alpha * (*this) * rhs.col(k) without transforms, + * for operators below the FFT threshold. */ template <typename Dest, typename Rhs, typename ProductScalar> void directProductColumn(Dest& dst, const Rhs& rhs, Index k, const ProductScalar& alpha) const { const Index m = rows(), n = cols(); @@ -350,7 +327,6 @@ ColGeneratorType m_col; RowGeneratorType m_row; ComplexVector m_symbol; - bool m_fftUsable; }; /** \ingroup StructuredMatrices_Module
diff --git a/contrib/Eigen/src/StructuredMatrices/Vandermonde.h b/contrib/Eigen/src/StructuredMatrices/Vandermonde.h index f218f09..b84f1cd 100644 --- a/contrib/Eigen/src/StructuredMatrices/Vandermonde.h +++ b/contrib/Eigen/src/StructuredMatrices/Vandermonde.h
@@ -11,10 +11,10 @@ // [1] N. J. Higham, "Accuracy and Stability of Numerical Algorithms", 2nd ed., // SIAM, 2002, chapter 27. Avoiding spurious overflow by rescaling with // powers of two, the technique behind determinant()'s balanced -// accumulation and the scaled Horner recurrence of addProduct(). +// accumulation. // [2] P. H. Sterbenz, "Floating-Point Computation", Prentice-Hall, 1974. // Scaling by a power of two is exact, the property the balanced -// accumulation and the scaled Horner recurrence rely on. +// accumulation relies on. // [3] J. J. Dongarra, J. R. Bunch, C. B. Moler and G. W. Stewart, "LINPACK // Users' Guide", SIAM, 1979. determinant()'s balanced accumulation follows // the convention of its xGEDI routines, which return determinants as a @@ -119,8 +119,8 @@ * \f[ (Va)_i = \sum_{j=0}^{n-1} a_j x_i^j, \qquad * p_{n-1}=a_{n-1},\quad p_j=a_j+x_i p_{j+1}. \f] * The class stores only the \c m nodes; products evaluate this Horner recurrence - * rule at O(mn) operations -- the same cost as a dense product, but with O(m) - * storage and without ever forming the matrix. + * at all nodes at once in O(mn) operations -- the same cost as a dense product, + * but with O(m) storage and without ever forming the matrix. * * Square systems are solved in O(n^2) by the Björck-Pereyra algorithm (class * \ref BjorckPereyra), whose \c transpose().solve() form covers the dual @@ -221,23 +221,17 @@ * products can neither overflow nor underflow when the determinant itself is * representable, whatever the spread of the nodes. Zero factors (repeated * nodes, giving an exactly singular matrix) and non-finite factors propagate - * exactly. */ + * exactly; a node difference that overflows makes the determinant infinite, or + * NaN when a zero factor is also present. */ Scalar determinant() const { eigen_assert(rows() == cols() && "Vandermonde::determinant requires a square matrix"); const Index n = rows(); Scalar det(1); internal::structured_exponent_type exponent = 0; for (Index j = 1; j < n; ++j) - for (Index i = 0; i < j; ++i) { - // A node difference can overflow even though the determinant is - // representable (e.g. nodes near +-max); the guarded difference then - // enters at half scale with the factor of two carried by the running - // exponent. - int shift; - const Scalar diff = internal::structured_guarded_diff(m_x.coeff(j), m_x.coeff(i), shift); - exponent += shift; - det = internal::structured_balance(det * internal::structured_balance(diff, exponent), exponent); - } + for (Index i = 0; i < j; ++i) + det = internal::structured_balance( + det * internal::structured_balance(Scalar(m_x.coeff(j) - m_x.coeff(i)), exponent), exponent); // ldexp saturates cleanly to zero / infinity once the accumulated exponent // leaves the representable range; the clamp only guards the narrowing to int. return internal::structured_ldexp_clamped(det, exponent); @@ -276,7 +270,7 @@ /** \returns the product expression \c (*this) * \a a: the polynomial with * ascending coefficients \c a (per column) evaluated at every node by Horner's - * rule, at O(mn) operations and O(1) extra storage. The expression carries the + * rule, at O(mn) operations and O(m) extra storage. The expression carries the * default product tag, so assigning it behaves like any dense product: a * temporary resolves aliasing between the destination and \a a, and * \c .noalias() skips it. */ @@ -289,162 +283,57 @@ return Product<Vandermonde, Rhs>(*this, a.derived()); } - /** \internal Computes \c dst += alpha * (*this) * rhs by Horner's rule. - * \c ProductScalar is the promoted scalar of the product (complex when a real - * operator is applied to a complex right-hand side); the accumulation runs in - * the promoted type. - * - * Horner intermediates can overflow even when the polynomial value itself is - * representable (e.g. coefficients near the overflow threshold evaluated at a - * node of magnitude 1/2). Each (node, column) pair is therefore screened with - * a conservative exponent bound: when no intermediate can overflow -- every - * input of moderate magnitude -- the plain Horner loop runs, matching the naive - * evaluation step for step, though not necessarily bit for bit: the compiler - * may contract one loop's multiply-add into an FMA and not the other's. - * Otherwise scaledHorner() keeps the running value in - * the balanced form m * 2^e of determinant(). Non-finite nodes or coefficients - * also take the plain loop, which propagates Inf/NaN entrywise like a dense - * product; and a unit alpha must not multiply -- even the identity complex - * scalar (1,0) pollutes an (Inf,0) value with NaN through the 0*Inf cross - * term. - * - * The column's finiteness and its exponent bound come from the single - * fast-max pass of internal::structured_exponent_bound_finite(), which is not - * guaranteed to propagate NaN. That is sufficient here for the same reason as - * in the FFT products: an Inf in NaN-free data always surfaces in a fast - * maximum, and a column containing NaN yields the NaN results dense-product - * semantics require through the plain loop and the scaled recurrence alike - * (every Horner step folds the NaN coefficient in, and the balancing helpers - * pass non-finite values through), so missing a NaN cannot change the - * result. */ + /** \internal Computes \c dst += alpha * (*this) * rhs by Horner's rule, + * vectorized across the nodes. \c ProductScalar is the promoted scalar of the + * product (complex when a real operator is applied to a complex right-hand + * side); the accumulation runs in the promoted type, and real nodes with + * complex values run one real recurrence per component. The recurrence runs + * in plain floating-point arithmetic, so an intermediate can overflow even + * when the polynomial value is representable. */ template <typename Dest, typename Rhs, typename ProductScalar> void addProduct(Dest& dst, const Rhs& rhs, const ProductScalar& alpha) const { - const Index m = rows(), n = m_cols; - using Exponent = internal::structured_exponent_type; - eigen_assert(rhs.rows() == n && "invalid product: dimensions do not match"); + eigen_assert(rhs.rows() == m_cols && "invalid product: dimensions do not match"); + // A unit alpha must not multiply: even the identity complex scalar (1,0) + // pollutes an (Inf,0) value with NaN through the 0*Inf cross term. const bool unitAlpha = alpha == ProductScalar(1); - int log2n = 0; // n < 2^log2n: bounds the number of addends of the Horner sum - for (Index t = n; t > 0; t /= 2) ++log2n; + using SplitComponents = + internal::bool_constant<!NumTraits<Scalar>::IsComplex && NumTraits<ProductScalar>::IsComplex>; + // Sized like the nodes: a fixed-size Index constructor would be ambiguous. + Array<ProductScalar, Rows_, 1> acc = m_x.array().template cast<ProductScalar>(); for (Index k = 0; k < rhs.cols(); ++k) { - int colExp; // max modulus < 2^colExp; 0 for a zero or non-finite column - const bool colFinite = internal::structured_exponent_bound_finite(rhs.col(k), colExp); - for (Index i = 0; i < m; ++i) { - const Scalar xi = m_x.coeff(i); - // |p_j| < 2^(colExp+log2n+(n-1) max(xiExp,0)+1). - const Exponent intermediateBound = - Exponent(colExp) + Exponent(log2n) + (Exponent(n) - 1) * Exponent(numext::maxi(exponentBound(xi), 0)) + 2; - const bool plain = !colFinite || !(numext::isfinite)(xi) || - intermediateBound <= Exponent(NumTraits<RealScalar>::max_exponent()); - ProductScalar acc; - if (plain) { - acc = rhs.coeff(n - 1, k); - for (Index j = n - 2; j >= 0; --j) acc = acc * xi + rhs.coeff(j, k); - } else { - acc = scaledHorner<ProductScalar>(xi, rhs, k); - } - dst.coeffRef(i, k) += unitAlpha ? acc : ProductScalar(alpha * acc); - } + horner(acc, rhs, k, SplitComponents()); + if (unitAlpha) + dst.col(k) += acc.matrix(); + else + dst.col(k) += alpha * acc.matrix(); } } private: - /** \internal \returns an exponent bound \c e with \c |z| < 2^e (the modulus - * for a complex \a z), or 0 for a zero or non-finite \a z; the - * single-coefficient analogue of internal::structured_exponent_bound(). */ - template <typename T> - static int exponentBound(const T& z) { - return exponentBoundImpl(z, internal::bool_constant<NumTraits<T>::IsComplex>()); - } - - template <typename T> - static int exponentBoundImpl(const T& z, std::false_type) { - if (!(numext::abs(z) > T(0)) || !(numext::isfinite)(z)) return 0; - int e; - EIGEN_USING_STD(frexp); - frexp(z, &e); - return e; - } - - template <typename T> - static int exponentBoundImpl(const T& z, std::true_type) { - using Real = typename NumTraits<T>::Real; - const Real mag = numext::maxi(numext::abs(numext::real(z)), numext::abs(numext::imag(z))); - if (!(mag > Real(0)) || !(numext::isfinite)(mag)) return 0; - int e; - EIGEN_USING_STD(frexp); - frexp(mag, &e); - return e + 1; // the modulus is at most sqrt(2) times the largest component - } - - /** \internal \returns \a z * 2^e computed through two exact half-factors, so - * the factors themselves stay representable for the exponent swings the scaled - * Horner recurrence produces (|e| up to about twice the scalar's exponent - * range on the negative side, at most one exponent range on the positive - * side); a factor past the underflow threshold flushes to zero together with - * the then-negligible contribution it scales. */ - template <typename T> - static T twoHalfScale(const T& z, internal::structured_exponent_type e) { - using Real = typename NumTraits<T>::Real; - constexpr internal::structured_exponent_type kMaxExponent = internal::structured_exponent_type(1) << 24; - const int ec = static_cast<int>(numext::mini(numext::maxi(e, -kMaxExponent), kMaxExponent)); - const Real h1 = numext::ldexp(Real(1), ec / 2); - const Real h2 = numext::ldexp(Real(1), ec - ec / 2); - return (z * h1) * h2; - } - - /** \internal Evaluates the polynomial with ascending coefficients - * \c rhs.col(k) at the node \a xi, keeping the running value in the balanced - * form \c acc * 2^exponent of determinant() (overflow-avoiding power-of-two - * rescaling [1], exact by [2]): the node enters through its unit mantissa - * with its exponent folded into the running one, the mantissa is - * renormalized after every step, and each coefficient is folded into the - * running frame scaled by an exact power of two split into two half-factors - * (when the coefficient dominates the frame, the frame is rebased onto the - * coefficient's exponent instead). Intermediates can therefore neither - * overflow nor underflow, and the final ldexp saturates to +-Inf / +-0 exactly - * where the true value leaves the representable range. - * - * An exactly zero mantissa carries no scale, so the frame is reset before - * every fold: after an exact cancellation (or a zero node annihilating the - * running value) a stale huge frame would otherwise underflow the next small - * coefficient to zero. A cancellation that leaves a tiny nonzero mantissa - * needs no such care -- the frexp renormalization rebases the frame to the - * surviving magnitude (which is a multiple of the operands' unit roundoff, - * hence never subnormal for real scalars, and frexp is exact on subnormal - * component values regardless). - * \pre the node is finite and the column passed the fast-max routing - * predicate of addProduct(): it holds no Inf without an accompanying NaN. A - * NaN-bearing column can reach the recurrence when the fast maximum misses - * the NaN; every helper passes non-finite values through, so it produces the - * NaN result dense-product semantics require. */ template <typename ProductScalar, typename Rhs> - ProductScalar scaledHorner(const Scalar& xi, const Rhs& rhs, Index k) const { - using Exponent = internal::structured_exponent_type; - Exponent xiE = 0; - const Scalar xiMant = internal::structured_balance(xi, xiE); // xi = xiMant * 2^xiE, exactly - ProductScalar acc(0); - Exponent exponent = 0; // running value = acc * 2^exponent - for (Index j = m_cols - 1; j >= 0; --j) { - if (j < m_cols - 1) { - exponent += xiE; - acc = internal::structured_balance(acc * xiMant, exponent); - } - // A zero value has no scale: reset the frame so the next coefficient - // enters at its own magnitude instead of underflowing in a stale one. - if (acc == ProductScalar(0)) exponent = 0; - const ProductScalar aj(rhs.coeff(j, k)); - if (aj == ProductScalar(0)) continue; - const Exponent ajExp = exponentBound(aj); - if (exponent < ajExp) { - // The coefficient dominates the running frame: rebase onto the - // coefficient's exponent. The running value rescales exactly, or - // underflows harmlessly once it is negligible against the coefficient. - acc = twoHalfScale(acc, exponent - ajExp); - exponent = ajExp; - } - acc = internal::structured_balance(acc + twoHalfScale(aj, -exponent), exponent); + void horner(Array<ProductScalar, Rows_, 1>& acc, const Rhs& rhs, Index k, std::false_type) const { + const Index n = m_cols; + acc.setConstant(ProductScalar(rhs.coeff(n - 1, k))); + for (Index j = n - 2; j >= 0; --j) + acc = acc * m_x.array().template cast<ProductScalar>() + ProductScalar(rhs.coeff(j, k)); + } + + // Complex values at real nodes: (re + i im) x keeps the exact + // real-times-complex arithmetic of a dense product, which the + // complex-times-complex form would break through 0 * Inf cross terms. + template <typename ProductScalar, typename Rhs> + void horner(Array<ProductScalar, Rows_, 1>& acc, const Rhs& rhs, Index k, std::true_type) const { + using NodeArray = Array<Scalar, Rows_, 1>; + const Index n = m_cols; + const ProductScalar top(rhs.coeff(n - 1, k)); + NodeArray re = NodeArray::Constant(rows(), numext::real(top)); + NodeArray im = NodeArray::Constant(rows(), numext::imag(top)); + for (Index j = n - 2; j >= 0; --j) { + const ProductScalar c(rhs.coeff(j, k)); + re = re * m_x.array() + numext::real(c); + im = im * m_x.array() + numext::imag(c); } - return internal::structured_ldexp_clamped(acc, exponent); + acc = re.binaryExpr(im, [](const Scalar& a, const Scalar& b) { return ProductScalar(a, b); }); } NodeVector m_x; @@ -626,21 +515,7 @@ #endif private: - static RealScalar lejaLogAbs(const Scalar& z) { - const RealScalar re = numext::abs(numext::real(z)); - const RealScalar im = numext::abs(numext::imag(z)); - const RealScalar scale = numext::maxi(re, im); - if (scale == RealScalar(0)) return -NumTraits<RealScalar>::infinity(); - const RealScalar scaledRe = re / scale; - const RealScalar scaledIm = im / scale; - return numext::log(scale) + RealScalar(0.5) * numext::log(scaledRe * scaledRe + scaledIm * scaledIm); - } - - static RealScalar lejaLogDistance(const Scalar& a, const Scalar& b) { - int exponent; - const Scalar difference = internal::structured_guarded_diff(a, b, exponent); - return lejaLogAbs(difference) + RealScalar(exponent) * numext::log(RealScalar(2)); - } + static RealScalar lejaLogAbs(const Scalar& z) { return numext::log(numext::abs(z)); } void initializeNodeOrder(const NodeVector&, std::false_type) {} @@ -668,7 +543,7 @@ RealScalar candidateScore = -NumTraits<RealScalar>::infinity(); for (Index i = 0; i < n; ++i) { if (selected[static_cast<std::size_t>(i)]) continue; - scores[static_cast<std::size_t>(i)] += lejaLogDistance(original[i], original[next]); + scores[static_cast<std::size_t>(i)] += lejaLogAbs(original[i] - original[next]); if (candidate < 0 || scores[static_cast<std::size_t>(i)] > candidateScore) { candidate = i; candidateScore = scores[static_cast<std::size_t>(i)];
diff --git a/contrib/Eigen/src/Tensor/README.md b/contrib/Eigen/src/Tensor/README.md index 45348bb..d0ca8ca 100644 --- a/contrib/Eigen/src/Tensor/README.md +++ b/contrib/Eigen/src/Tensor/README.md
@@ -598,8 +598,7 @@ and stops the program, through `std::abort()` when `EIGEN_NO_DEBUG` (or `NDEBUG`) is defined and a failed `eigen_assert` otherwise. Define `EIGEN_GPU_RUNTIME_CHECK(call)` before including the Tensor header to route -failures elsewhere, `EIGEN_GPU_NO_RUNTIME_CHECKS` to ignore them as release -builds of earlier versions did, or `EIGEN_GPU_SYNC_LAUNCHES` to synchronize +failures elsewhere, or `EIGEN_GPU_SYNC_LAUNCHES` to synchronize after every kernel launch so that an execution failure is reported at the launch that caused it.
diff --git a/contrib/Eigen/src/Tensor/TensorRandom.h b/contrib/Eigen/src/Tensor/TensorRandom.h index a9ef7cf..14266fe 100644 --- a/contrib/Eigen/src/Tensor/TensorRandom.h +++ b/contrib/Eigen/src/Tensor/TensorRandom.h
@@ -190,7 +190,9 @@ const T x = u - T(0.449871); const T y = numext::abs(v) + T(0.386595); q = x * x + y * (T(0.196) * y - T(0.25472) * x); - } while (q > T(0.27597) && (q > T(0.27846) || v * v > T(-4) * numext::log(u) * u * u)); + // u == 0 is outside the acceptance region (q > 0.27597 there), but -4 * log(0) * 0 * 0 is NaN, so the last test + // would accept it and return v / 0 = +/-inf. + } while (u == T(0) || (q > T(0.27597) && (q > T(0.27846) || v * v > T(-4) * numext::log(u) * u * u))); return v / u; }
diff --git a/contrib/benchmarks/GPU/CMakeLists.txt b/contrib/benchmarks/GPU/CMakeLists.txt index b53c2a4..a2ddef4 100644 --- a/contrib/benchmarks/GPU/CMakeLists.txt +++ b/contrib/benchmarks/GPU/CMakeLists.txt
@@ -54,7 +54,7 @@ eigen_add_gpu_benchmark(bench_gemm_float bench_gemm.cpp LIBRARIES CUDA::cublasLt DEFINITIONS SCALAR=float) # Overhead micro-benchmarks: per-call allocation, hidden syncs, host-side copies. -eigen_add_gpu_benchmark(bench_overhead bench_overhead.cpp LIBRARIES CUDA::cublasLt CUDA::cusparse) +eigen_add_gpu_benchmark(bench_overhead bench_overhead.cpp LIBRARIES CUDA::cublasLt CUDA::cusparse CUDA::npps CUDA::nppc) # Unified/zero-copy memory: storage strategies for integrated GPUs (Jetson). eigen_add_gpu_benchmark(bench_unified_memory bench_unified_memory.cpp)
diff --git a/contrib/benchmarks/GPU/bench_overhead.cpp b/contrib/benchmarks/GPU/bench_overhead.cpp index b5d42ba..decbcc5 100644 --- a/contrib/benchmarks/GPU/bench_overhead.cpp +++ b/contrib/benchmarks/GPU/bench_overhead.cpp
@@ -17,6 +17,8 @@ // FromHost / FromHostAsync — extra host-side PlainMatrix copy in fromHost() // GemmFreshDst / PreallocDst — cudaMalloc/cudaFree per GEMM temporary // DotDeviceScalar / DotRaw — DeviceScalar wrapper cost per reduction +// DotIntoDeviceScalar — the same reduction into an existing DeviceScalar +// NormRead / StableNormRead — norm() (dot + NPP sqrt) vs stableNorm() (nrm2), read back // OneShotLltExpr / CachedLlt / RawPotrs — expression-solve sync + allocs // CudaMalloc / CudaMallocAsync — stream-ordered allocation as a remedy // SmallBuffer* / PoolAllocFree — small DeviceBuffer round trip (idle, behind in-flight work) and the pool itself @@ -164,6 +166,50 @@ } BENCHMARK(BM_DotRawCublas)->Arg(1 << 12)->Arg(1 << 20)->UseRealTime()->MinWarmUpTime(0.5); +// The reduction into a DeviceScalar allocated once: no allocation per call. +static void BM_DotIntoDeviceScalar(benchmark::State& state) { + const Index n = state.range(0); + gpu::Context& ctx = gpu::Context::threadLocal(); + DeviceMatrix d_x = DeviceMatrix::fromHost(HostMatrix::Random(n, 1), ctx.stream()); + DeviceMatrix d_y = DeviceMatrix::fromHost(HostMatrix::Random(n, 1), ctx.stream()); + gpu::DeviceScalar<Scalar> r(ctx.stream()); + for (auto _ : state) { + d_x.dot(ctx, d_y, r); + benchmark::DoNotOptimize(r.devicePtr()); + syncStream(ctx.stream()); + } +} +BENCHMARK(BM_DotIntoDeviceScalar)->Arg(1 << 12)->Arg(1 << 20)->UseRealTime()->MinWarmUpTime(0.5); + +// norm() is a dot and a one-element NPP sqrt; stableNorm() is cuBLAS nrm2's +// scaled accumulation. Each iteration reads the norm back to the host, as the +// convergence test of an iterative method does. +static void BM_NormRead(benchmark::State& state) { + const Index n = state.range(0); + gpu::Context& ctx = gpu::Context::threadLocal(); + DeviceMatrix d_x = DeviceMatrix::fromHost(HostMatrix::Random(n, 1), ctx.stream()); + gpu::DeviceScalar<Scalar> r(ctx.stream()); + for (auto _ : state) { + d_x.norm(ctx, r); + Scalar value = r; + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_NormRead)->Arg(1 << 12)->Arg(1 << 20)->UseRealTime()->MinWarmUpTime(0.5); + +static void BM_StableNormRead(benchmark::State& state) { + const Index n = state.range(0); + gpu::Context& ctx = gpu::Context::threadLocal(); + DeviceMatrix d_x = DeviceMatrix::fromHost(HostMatrix::Random(n, 1), ctx.stream()); + gpu::DeviceScalar<Scalar> r(ctx.stream()); + for (auto _ : state) { + d_x.stableNorm(ctx, r); + Scalar value = r; + benchmark::DoNotOptimize(value); + } +} +BENCHMARK(BM_StableNormRead)->Arg(1 << 12)->Arg(1 << 20)->UseRealTime()->MinWarmUpTime(0.5); + // --------------------------------------------------------------------------- // 4. Cholesky solve paths. // OneShotLltExpr: d_X = d_A.llt().solve(d_B) — re-factorizes, allocates
diff --git a/contrib/benchmarks/StructuredMatrices/CMakeLists.txt b/contrib/benchmarks/StructuredMatrices/CMakeLists.txt index aa3f795..fa4922d 100644 --- a/contrib/benchmarks/StructuredMatrices/CMakeLists.txt +++ b/contrib/benchmarks/StructuredMatrices/CMakeLists.txt
@@ -17,3 +17,4 @@ eigen_add_benchmark(bench_structured_circulant_inverse bench_structured_circulant_inverse.cpp) eigen_add_benchmark(bench_structured_materialize bench_structured_materialize.cpp) eigen_add_benchmark(bench_structured_kronecker_spectral bench_structured_kronecker_spectral.cpp) +eigen_add_benchmark(bench_structured_kronecker_batched bench_structured_kronecker_batched.cpp)
diff --git a/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker.cpp b/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker.cpp index ca9590a..690a672 100644 --- a/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker.cpp +++ b/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker.cpp
@@ -58,9 +58,9 @@ BENCHMARK(BM_KroneckerMaterializeDense)->Arg(8)->Arg(16)->Arg(32)->Arg(64); // --- Multiple right-hand sides --- -// Both the product and the direct solve walk the right-hand side column by -// column through the vec identity, so these cover the path where the per-column -// vec-trick workspaces are reused instead of reallocated. +// The product and the direct solve apply the factors to cache-sized batches of +// right-hand sides (see bench_structured_kronecker_batched for the sweep over +// the number of right-hand sides). static void BM_KroneckerProductImplicitMultiRhs(benchmark::State& state) { const Index n = state.range(0), nrhs = state.range(1); Mat A = Mat::Random(n, n), B = Mat::Random(n, n); @@ -140,8 +140,8 @@ BENCHMARK(BM_KroneckerProductIdentityRightDiag)->Arg(8)->Arg(16)->Arg(32)->Arg(64); // Solving (D (x) B) x = b: a densely stored diagonal factor costs a full LU -// per solve call; the DiagonalMatrix factor is normalized once and divided -// entrywise, leaving the single LU of the dense factor. +// per solve call; the DiagonalMatrix factor is divided entrywise, leaving the +// single LU of the dense factor. static void BM_KroneckerSolveDiagFactorDense(benchmark::State& state) { const Index n = state.range(0); Vec d = Vec::Random(n) + Vec::Constant(n, 2.0); @@ -228,7 +228,7 @@ benchmark::DoNotOptimize(y.data()); } } -BENCHMARK(BM_KroneckerProductIdentityLeftSparse)->Arg(64)->Arg(128)->Arg(256)->Arg(512); +BENCHMARK(BM_KroneckerProductIdentityLeftSparse)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024); static void BM_KroneckerProductIdentityLeftSparseMaterialized(benchmark::State& state) { const Index n = state.range(0); @@ -241,7 +241,7 @@ benchmark::DoNotOptimize(y.data()); } } -BENCHMARK(BM_KroneckerProductIdentityLeftSparseMaterialized)->Arg(64)->Arg(128)->Arg(256)->Arg(512); +BENCHMARK(BM_KroneckerProductIdentityLeftSparseMaterialized)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024); static void BM_KroneckerProductIdentityRightSparse(benchmark::State& state) { const Index n = state.range(0); @@ -253,7 +253,7 @@ benchmark::DoNotOptimize(y.data()); } } -BENCHMARK(BM_KroneckerProductIdentityRightSparse)->Arg(64)->Arg(128)->Arg(256)->Arg(512); +BENCHMARK(BM_KroneckerProductIdentityRightSparse)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024); static void BM_KroneckerProductIdentityRightSparseMaterialized(benchmark::State& state) { const Index n = state.range(0); @@ -266,7 +266,7 @@ benchmark::DoNotOptimize(y.data()); } } -BENCHMARK(BM_KroneckerProductIdentityRightSparseMaterialized)->Arg(64)->Arg(128)->Arg(256)->Arg(512); +BENCHMARK(BM_KroneckerProductIdentityRightSparseMaterialized)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024); static void BM_KroneckerMaterializeSparse(benchmark::State& state) { const Index n = state.range(0); @@ -332,24 +332,85 @@ } BENCHMARK(BM_KroneckerSolveIdentityRightSparseMaterialized)->Arg(64)->Arg(128)->Arg(256); -static void BM_KroneckerProductNonFiniteSparse(benchmark::State& state) { +// I_n (x) A and A (x) I_n with the identity passed as Identity(): its side of +// the product is skipped, so a single right-hand side costs one sparse-dense +// product over x reshaped in place -- the work of the materialized product +// without storing it. The unit-diagonal cases above pay one diagonal scaling +// and one temporary on top. +static void BM_KroneckerProductIdentityFactorLeftSparse(benchmark::State& state) { const Index n = state.range(0); - SpMat A(n, n); - for (Index i = 1; i < n; ++i) A.insert(i, i) = 2.0; - Mat B = Mat::Identity(2, 2); - B(0, 0) = NumTraits<double>::infinity(); - auto K = makeKroneckerOperator(B, A); - Vec x = Vec::Ones(2 * n), y = K * x; - Vec expected = Vec::Constant(2 * n, 2.0); - expected.head(n).setConstant(NumTraits<double>::infinity()); - expected[0] = expected[n] = 0.0; // The empty sparse row annihilates Inf. - if (!(y.array() == expected.array()).all()) { - state.SkipWithError("non-finite product did not preserve structural zeros"); - return; - } + SpMat A = tridiagonal(n); + auto K = makeKroneckerOperator(Mat::Identity(n, n), A); + Vec x = Vec::Random(n * n), y(n * n); for (auto _ : state) { y.noalias() = K * x; benchmark::DoNotOptimize(y.data()); } } -BENCHMARK(BM_KroneckerProductNonFiniteSparse)->Arg(64)->Arg(128)->Arg(256)->Arg(512); +BENCHMARK(BM_KroneckerProductIdentityFactorLeftSparse)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024); + +static void BM_KroneckerProductIdentityFactorRightSparse(benchmark::State& state) { + const Index n = state.range(0); + SpMat A = tridiagonal(n); + auto K = makeKroneckerOperator(A, Mat::Identity(n, n)); + Vec x = Vec::Random(n * n), y(n * n); + for (auto _ : state) { + y.noalias() = K * x; + benchmark::DoNotOptimize(y.data()); + } +} +BENCHMARK(BM_KroneckerProductIdentityFactorRightSparse)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024); + +// The middle term I_n (x) A (x) I_n of a 3-D finite-difference operator on an +// n^3 grid: nested Kronecker factors against the materialized sparse product. +static void BM_KroneckerProductNestedSandwichSparse(benchmark::State& state) { + const Index n = state.range(0); + SpMat A = tridiagonal(n); + auto K = makeKroneckerOperator(Mat::Identity(n, n), A, Mat::Identity(n, n)); + Vec x = Vec::Random(n * n * n), y(n * n * n); + for (auto _ : state) { + y.noalias() = K * x; + benchmark::DoNotOptimize(y.data()); + } +} +BENCHMARK(BM_KroneckerProductNestedSandwichSparse)->Arg(16)->Arg(32)->Arg(64)->Arg(128); + +static void BM_KroneckerProductNestedSandwichSparseMaterialized(benchmark::State& state) { + const Index n = state.range(0); + SpMat A = tridiagonal(n); + SpMat K; + K = makeKroneckerOperator(Mat::Identity(n, n), A, Mat::Identity(n, n)); + Vec x = Vec::Random(n * n * n), y(n * n * n); + for (auto _ : state) { + y.noalias() = K * x; + benchmark::DoNotOptimize(y.data()); + } +} +BENCHMARK(BM_KroneckerProductNestedSandwichSparseMaterialized)->Arg(16)->Arg(32)->Arg(64)->Arg(128); + +// The same operator nested to the left, (I_n (x) A) (x) I_n, which applies the +// nested factor from the right. +static void BM_KroneckerProductLeftNestedSandwichSparse(benchmark::State& state) { + const Index n = state.range(0); + SpMat A = tridiagonal(n); + auto K = makeKroneckerOperator(makeKroneckerOperator(Mat::Identity(n, n), A), Mat::Identity(n, n)); + Vec x = Vec::Random(n * n * n), y(n * n * n); + for (auto _ : state) { + y.noalias() = K * x; + benchmark::DoNotOptimize(y.data()); + } +} +BENCHMARK(BM_KroneckerProductLeftNestedSandwichSparse)->Arg(16)->Arg(32)->Arg(64)->Arg(128); + +// (A (x) A) (x) A on r right-hand sides, no identity factor. +static void BM_KroneckerProductLeftNestedSparse(benchmark::State& state) { + const Index n = state.range(0), r = state.range(1); + SpMat A = tridiagonal(n); + auto K = makeKroneckerOperator(makeKroneckerOperator(A, A), A); + Mat X = Mat::Random(n * n * n, r), Y(n * n * n, r); + for (auto _ : state) { + Y.noalias() = K * X; + benchmark::DoNotOptimize(Y.data()); + } +} +BENCHMARK(BM_KroneckerProductLeftNestedSparse)->ArgsProduct({{16, 32, 64}, {1, 8}});
diff --git a/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker_batched.cpp b/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker_batched.cpp new file mode 100644 index 0000000..c06b76e --- /dev/null +++ b/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker_batched.cpp
@@ -0,0 +1,159 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +// KroneckerOperator products and solves, (A (x) B) vec(X) = vec(B X A^T), with +// n x n factors and r right-hand sides. The product GFLOPS count +// 2 r (work(B) n1 + m2 work(A)) flops, work(F) = stored entries of F. Each +// benchmark first checks its result outside the timed loop. + +#include <benchmark/benchmark.h> +#include <Eigen/Dense> +#include <Eigen/Sparse> +#include <contrib/Eigen/StructuredMatrices> + +using namespace Eigen; + +using Mat = MatrixXd; +using SpMat = SparseMatrix<double>; +using Diag = DiagonalMatrix<double, Dynamic>; + +namespace { + +// The 1-D Laplacian tridiag(-1, 2, -1): its Kronecker sums are the 2-D +// finite-difference operators. +SpMat laplacian(Index n) { + SpMat L(n, n); + L.reserve(VectorXi::Constant(n, 3)); + for (Index j = 0; j < n; ++j) { + if (j > 0) L.insert(j - 1, j) = -1.0; + L.insert(j, j) = 2.0; + if (j + 1 < n) L.insert(j + 1, j) = -1.0; + } + L.makeCompressed(); + return L; +} + +Mat dominant(Index n) { return Mat::Random(n, n) + double(2 * n) * Mat::Identity(n, n); } + +void setFlops(benchmark::State& state, double flopsPerIteration) { + state.counters["GFLOPS"] = + benchmark::Counter(1e-9 * flopsPerIteration, benchmark::Counter::kIsIterationInvariantRate); +} + +// The batched result must match the operator applied to one column at a time. +template <typename Lhs, typename Rhs> +bool productMatchesColumns(const KroneckerOperator<Lhs, Rhs>& K, const Mat& X, const Mat& Y) { + Mat expected(K.rows(), X.cols()); + for (Index k = 0; k < X.cols(); ++k) expected.col(k) = K * X.col(k); + return (Y - expected).norm() <= 1e-10 * expected.norm(); +} + +// K x = b up to the conditioning of the (diagonally dominant) factors. +template <typename Lhs, typename Rhs> +bool solveHasSmallResidual(const KroneckerOperator<Lhs, Rhs>& K, const Mat& x, const Mat& b) { + const Mat residual = K * x - b; + return residual.norm() <= 1e-8 * b.norm(); +} + +template <typename Lhs, typename Rhs> +void runProduct(benchmark::State& state, const KroneckerOperator<Lhs, Rhs>& K, double workA, double workB) { + const Index r = state.range(1); + const Mat X = Mat::Random(K.cols(), r); + Mat Y = K * X; + if (!productMatchesColumns(K, X, Y)) { + state.SkipWithError("batched product differs from the column-by-column product"); + return; + } + for (auto _ : state) { + Y.noalias() = K * X; + benchmark::DoNotOptimize(Y.data()); + benchmark::ClobberMemory(); + } + const double n1 = double(K.lhs().cols()), m2 = double(K.rhs().rows()); + setFlops(state, 2.0 * double(r) * (workB * n1 + m2 * workA)); +} + +void BM_ProductDense(benchmark::State& state) { + const Index n = state.range(0); + const KroneckerOperator<Mat, Mat> K(Mat::Random(n, n), Mat::Random(n, n)); + runProduct(state, K, double(n * n), double(n * n)); +} + +void BM_ProductSparse(benchmark::State& state) { + const Index n = state.range(0); + const SpMat L = laplacian(n); + const KroneckerOperator<SpMat, SpMat> K(L, L); + runProduct(state, K, double(L.nonZeros()), double(L.nonZeros())); +} + +// I_n (x) B, the block-diagonal operator of n independent n x n systems. +void BM_ProductIdentityDense(benchmark::State& state) { + const Index n = state.range(0); + const KroneckerOperator<Diag, Mat> K(Diag(VectorXd::Ones(n)), Mat::Random(n, n)); + runProduct(state, K, double(n), double(n * n)); +} + +// L (x) I_n and I_n (x) L: a sparse factor against a diagonal one. +void BM_ProductSparseIdentity(benchmark::State& state) { + const SpMat L = laplacian(state.range(0)); + const KroneckerOperator<SpMat, Diag> K(L, Diag(VectorXd::Ones(L.rows()))); + runProduct(state, K, double(L.nonZeros()), double(L.rows())); +} + +void BM_ProductIdentitySparse(benchmark::State& state) { + const SpMat L = laplacian(state.range(0)); + const KroneckerOperator<Diag, SpMat> K(Diag(VectorXd::Ones(L.rows())), L); + runProduct(state, K, double(L.rows()), double(L.nonZeros())); +} + +template <typename Lhs, typename Rhs> +void runSolve(benchmark::State& state, const KroneckerOperator<Lhs, Rhs>& K) { + const Mat b = Mat::Random(K.rows(), state.range(1)); + Mat x = K.solve(b); + if (!solveHasSmallResidual(K, x, b)) { + state.SkipWithError("solve residual too large"); + return; + } + for (auto _ : state) { + x = K.solve(b); + benchmark::DoNotOptimize(x.data()); + benchmark::ClobberMemory(); + } +} + +void BM_SolveDense(benchmark::State& state) { + const Index n = state.range(0); + runSolve(state, KroneckerOperator<Mat, Mat>(dominant(n), dominant(n))); +} + +void BM_SolveSparse(benchmark::State& state) { + const SpMat L = laplacian(state.range(0)); + runSolve(state, KroneckerOperator<SpMat, SpMat>(L, L)); +} + +void BM_LeastSquaresDense(benchmark::State& state) { + const Index n = state.range(0); + const KroneckerOperator<Mat, Mat> K(dominant(n), dominant(n)); // full rank: least squares is the solve + const Mat b = Mat::Random(K.rows(), state.range(1)); + Mat x = K.leastSquaresSolve(b); + if (!solveHasSmallResidual(K, x, b)) { + state.SkipWithError("least-squares residual too large"); + return; + } + for (auto _ : state) { + x = K.leastSquaresSolve(b); + benchmark::DoNotOptimize(x.data()); + benchmark::ClobberMemory(); + } +} + +} // namespace + +BENCHMARK(BM_ProductDense)->ArgsProduct({{8, 32, 128}, {1, 32}}); +BENCHMARK(BM_ProductSparse)->ArgsProduct({{64, 512}, {1, 32}}); +BENCHMARK(BM_ProductIdentityDense)->ArgsProduct({{8, 32, 128}, {1, 32}}); +BENCHMARK(BM_ProductSparseIdentity)->ArgsProduct({{64, 512}, {1, 32}}); +BENCHMARK(BM_ProductIdentitySparse)->ArgsProduct({{64, 512}, {1, 32}}); +BENCHMARK(BM_SolveDense)->ArgsProduct({{8, 32, 128}, {1, 32}}); +BENCHMARK(BM_SolveSparse)->ArgsProduct({{64, 512}, {1, 32}}); +BENCHMARK(BM_LeastSquaresDense)->ArgsProduct({{32}, {1, 32}});
diff --git a/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker_spectral.cpp b/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker_spectral.cpp index 66d5521..e7d3847 100644 --- a/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker_spectral.cpp +++ b/contrib/benchmarks/StructuredMatrices/bench_structured_kronecker_spectral.cpp
@@ -29,7 +29,7 @@ benchmark::ClobberMemory(); } } -BENCHMARK_TEMPLATE(BM_KroneckerLeastSquaresMultiRhs, double)->ArgsProduct({{8, 16, 32}, {1, 8, 64}}); +BENCHMARK_TEMPLATE(BM_KroneckerLeastSquaresMultiRhs, double)->ArgsProduct({{8, 16, 32, 64}, {1, 8, 64}}); BENCHMARK_TEMPLATE(BM_KroneckerLeastSquaresMultiRhs, std::complex<double>)->ArgsProduct({{8, 16}, {1, 8, 64}}); template <typename Scalar>
diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 61ea341..102eec4 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt
@@ -38,6 +38,7 @@ ei_add_test(NonLinearOptimization) ei_add_test(NumericalDiff) +ei_add_test(stencil) ei_add_test(structured_dpr1) ei_add_test(autodiff_scalar) ei_add_test(autodiff)
diff --git a/contrib/test/EulerAngles.cpp b/contrib/test/EulerAngles.cpp index 150e7d2..20062ce 100644 --- a/contrib/test/EulerAngles.cpp +++ b/contrib/test/EulerAngles.cpp
@@ -289,6 +289,19 @@ VERIFY((m - e.toRotationMatrix()).norm() < 1e-6); } +template <typename Scalar> +void eulerangles_data_constructor() { + using EulerAnglesType = EulerAngles<Scalar, EulerSystemXYZ>; + const auto alpha = Scalar{123.4567}; + const auto beta = Scalar{-98.7654}; + const auto gamma = Scalar{4856.2954}; + const Scalar data[]{alpha, beta, gamma}; + const EulerAnglesType e(data); + VERIFY_IS_APPROX(e.alpha(), alpha); + VERIFY_IS_APPROX(e.beta(), beta); + VERIFY_IS_APPROX(e.gamma(), gamma); +} + EIGEN_DECLARE_TEST(EulerAngles) { // Simple cast test EulerAnglesXYZd onesEd(1, 1, 1); @@ -310,6 +323,9 @@ CALL_SUBTEST_5(eulerangles_gimbal_lock_regression<double>()); CALL_SUBTEST_5(eulerangles_gimbal_lock_reproducer()); + CALL_SUBTEST_6(eulerangles_data_constructor<float>()); + CALL_SUBTEST_6(eulerangles_data_constructor<double>()); + // TODO: Add tests for auto diff // TODO: Add tests for complex numbers }
diff --git a/contrib/test/GPU/CMakeLists.txt b/contrib/test/GPU/CMakeLists.txt index b0f9437..e7e66c7 100644 --- a/contrib/test/GPU/CMakeLists.txt +++ b/contrib/test/GPU/CMakeLists.txt
@@ -86,14 +86,14 @@ find_package(Threads REQUIRED) ei_add_gpu_test(device_buffer_pool EXTRA_LIBS Threads::Threads) -# cuBLAS integration. OFF by default; later MRs in the stack opt in. -option(EIGEN_TEST_CUBLAS "Test cuBLAS integration" OFF) +# cuBLAS integration. +option(EIGEN_TEST_CUBLAS "Test cuBLAS integration" ON) if(EIGEN_TEST_CUBLAS AND TARGET CUDA::cublas) ei_add_gpu_test(cublas EXTRA_LIBS CUDA::cublas CUDA::cusolver) endif() # cuSOLVER dense factorizations (LLT, LU, QR, SVD, self-adjoint eigensolver). -option(EIGEN_TEST_CUSOLVER "Test cuSOLVER integration" OFF) +option(EIGEN_TEST_CUSOLVER "Test cuSOLVER integration" ON) if(EIGEN_TEST_CUSOLVER AND TARGET CUDA::cusolver) find_package(Threads REQUIRED) ei_add_gpu_test(cusolver_llt EXTRA_LIBS CUDA::cusolver CUDA::cublas Threads::Threads) @@ -119,6 +119,14 @@ EXTRA_LIBS CUDA::cusparse CUDA::cublas CUDA::cusolver CUDA::npps CUDA::nppc) endif() +# Failed runtime and library calls reach EIGEN_GPU_CHECK_FAILED, with and without EIGEN_NO_DEBUG. +if(TARGET CUDA::cusparse AND TARGET CUDA::cufft) + foreach(_check_test runtime_checks runtime_checks_ndebug) + ei_add_gpu_test(${_check_test} + EXTRA_LIBS CUDA::cublas CUDA::cublasLt CUDA::cusolver CUDA::cusparse CUDA::cufft CUDA::npps CUDA::nppc) + endforeach() +endif() + # cuDSS sparse direct solvers -- distributed separately from the CUDA Toolkit. option(EIGEN_TEST_CUDSS "Test cuDSS sparse solver integration" OFF) if(EIGEN_TEST_CUDSS)
diff --git a/contrib/test/GPU/cg.cpp b/contrib/test/GPU/cg.cpp index 4383ec2..24949cd 100644 --- a/contrib/test/GPU/cg.cpp +++ b/contrib/test/GPU/cg.cpp
@@ -8,10 +8,10 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. // SPDX-License-Identifier: MPL-2.0 -// End-to-end test: CG algorithm running on GPU via gpu::DeviceMatrix. -// -// Uses DeviceSparseView for SpMV, gpu::DeviceMatrix for vectors, DeviceScalar -// for deferred reductions. Verifies correctness against CPU ConjugateGradient. +// End-to-end tests of conjugate gradient on the GPU types: Eigen's ConjugateGradient +// class on DeviceSparseView and DeviceMatrix through solveWithGuessInPlace(), and the +// module's hand-written loop with DeviceScalar reductions. Both are checked against +// the CPU ConjugateGradient. #define EIGEN_USE_GPU #include "main.h" @@ -22,8 +22,9 @@ using namespace Eigen; -// ---- Helper: build a sparse SPD matrix -------------------------------------- +// ---- Helpers ---------------------------------------------------------------- +// A = R^T R + n I: lambda_min >= n and lambda_max <= ||A||_F, so kappa(A) <= ||A||_F / n. template <typename Scalar> SparseMatrix<Scalar, ColMajor, int> make_spd(Index n, double density = 0.1) { using SpMat = SparseMatrix<Scalar, ColMajor, int>; @@ -45,118 +46,157 @@ return A; } -// ---- GPU CG without preconditioner ------------------------------------------ +template <typename Scalar> +double condition_bound(const SparseMatrix<Scalar, ColMajor, int>& A) { + return double(A.norm()) / double(A.rows()); +} + +// The recursively updated residual keeps shrinking below eps, so a tolerance under eps +// still reports convergence while certifying nothing about x. +template <typename Scalar> +typename NumTraits<Scalar>::Real cg_tolerance() { + return typename NumTraits<Scalar>::Real(100) * NumTraits<Scalar>::epsilon(); +} template <typename Scalar> -void test_gpu_cg(Index n) { +Matrix<Scalar, Dynamic, 1> cpu_reference(const SparseMatrix<Scalar, ColMajor, int>& A, + const Matrix<Scalar, Dynamic, 1>& b, typename NumTraits<Scalar>::Real tol) { + ConjugateGradient<SparseMatrix<Scalar, ColMajor, int>, Lower | Upper, IdentityPreconditioner> cpu_cg; + cpu_cg.setMaxIterations(1000); + cpu_cg.setTolerance(tol); + cpu_cg.compute(A); + Matrix<Scalar, Dynamic, 1> x = cpu_cg.solve(b); + VERIFY_IS_EQUAL(cpu_cg.info(), Success); + return x; +} + +// At exit the recursive residual is below tol ||b||; the true residual differs by the +// rounding of k <~ 10 iterations of SpMV and vector updates, ~ k sqrt(n) eps ||b|| for random +// signs. Both solutions satisfy that bound, so +// ||x_gpu - x_cpu|| <= ||A^-1|| (||r_gpu|| + ||r_cpu||) <= 2 kappa relres_bound ||x_cpu||. +// The residuals are evaluated in double so that a float check carries no rounding of its own. +template <typename Scalar> +void check_cg_solution(const SparseMatrix<Scalar, ColMajor, int>& A, const Matrix<Scalar, Dynamic, 1>& b, + const Matrix<Scalar, Dynamic, 1>& x_gpu, const Matrix<Scalar, Dynamic, 1>& x_cpu, + typename NumTraits<Scalar>::Real tol) { + const double eps = double(NumTraits<Scalar>::epsilon()); + const double relres_bound = double(tol) + 10 * std::sqrt(double(A.rows())) * eps; + const VectorXd r = A.template cast<double>() * x_gpu.template cast<double>() - b.template cast<double>(); + VERIFY(r.norm() <= relres_bound * b.template cast<double>().norm()); + const VectorXd dx = (x_gpu - x_cpu).template cast<double>(); + VERIFY(dx.norm() <= 2 * condition_bound(A) * relres_bound * x_cpu.template cast<double>().norm()); +} + +// ---- Eigen's ConjugateGradient class on device types ------------------------ +// DeviceSparseView is a matrix-free matrix type, so the public solver class +// instantiates with it; solveWithGuessInPlace() is the entry point that takes +// non-Eigen vectors. +template <typename Scalar> +void test_gpu_cg_class(Index n) { using SpMat = SparseMatrix<Scalar, ColMajor, int>; using Vec = Matrix<Scalar, Dynamic, 1>; using RealScalar = typename NumTraits<Scalar>::Real; - SpMat A = make_spd<Scalar>(n); Vec b = Vec::Random(n); + const RealScalar tol = cg_tolerance<Scalar>(); + Vec x_cpu = cpu_reference(A, b, tol); - // CPU reference (identity preconditioner to match GPU). - ConjugateGradient<SpMat, Lower | Upper, IdentityPreconditioner> cpu_cg; - cpu_cg.setMaxIterations(1000); - cpu_cg.setTolerance(RealScalar(1e-8)); - cpu_cg.compute(A); - Vec x_cpu = cpu_cg.solve(b); - VERIFY_IS_EQUAL(cpu_cg.info(), Success); - - // GPU CG: mirrors Eigen's conjugate_gradient() using gpu::DeviceMatrix ops. gpu::Context ctx; gpu::Context::setThreadLocal(&ctx); gpu::SparseContext<Scalar> spmv_ctx(ctx); auto mat = spmv_ctx.deviceView(A); - auto d_b = gpu::DeviceMatrix<Scalar>::fromHost(b, ctx.stream()); gpu::DeviceMatrix<Scalar> d_x(n, 1); d_x.setZero(ctx); - // r = b (since x=0) - gpu::DeviceMatrix<Scalar> residual(n, 1); - residual.copyFrom(ctx, d_b); + ConjugateGradient<gpu::DeviceSparseView<Scalar>, Lower | Upper, IdentityPreconditioner> cg; + cg.setMaxIterations(1000); + cg.setTolerance(tol); + cg.compute(mat); + cg.solveWithGuessInPlace(d_b, d_x); + VERIFY_IS_EQUAL(cg.info(), Success); + VERIFY(cg.iterations() > 0 && cg.iterations() < 1000); + VERIFY(cg.error() <= tol); - RealScalar rhsNorm2 = d_b.squaredNorm(ctx); - RealScalar tol = RealScalar(1e-8); - RealScalar threshold = tol * tol * rhsNorm2; - RealScalar residualNorm2 = residual.squaredNorm(ctx); - - // p = r (no preconditioner) - gpu::DeviceMatrix<Scalar> p(n, 1); - p.copyFrom(ctx, residual); - gpu::DeviceMatrix<Scalar> z(n, 1), tmp(n, 1); - - auto absNew = residual.dot(ctx, p); - Index maxIters = 1000; - Index i = 0; - while (i < maxIters) { - tmp.noalias() = mat * p; - - auto alpha = absNew / p.dot(ctx, tmp); - d_x += alpha * p; - residual -= alpha * tmp; - - residualNorm2 = residual.squaredNorm(ctx); - if (residualNorm2 < threshold) break; - - // z = r (no preconditioner) - z.copyFrom(ctx, residual); - - auto absOld = std::move(absNew); - absNew = residual.dot(ctx, z); - auto beta = absNew / absOld; - - p *= beta; - p += z; - i++; - } - + // The deep copy and the scalar division the algorithm relies on (`p = precond.solve(r)`, + // `residual /= residualScale`). 3 is not a power of two, so the quotient rounds; each element + // must be within 2 eps of the host division. + gpu::DeviceMatrix<Scalar> d_c = d_b; + d_c /= Scalar(3); gpu::Context::setThreadLocal(nullptr); + const Vec c = d_c.toHost(ctx.stream()); + const Vec c_ref = b / Scalar(3); + VERIFY(((c - c_ref).cwiseAbs().array() <= RealScalar(2) * NumTraits<Scalar>::epsilon() * c_ref.cwiseAbs().array()) + .all()); Vec x_gpu = d_x.toHost(ctx.stream()); - - // Verify residual. - Vec r = A * x_gpu - b; - RealScalar relres = r.norm() / b.norm(); - VERIFY(relres < RealScalar(1e-6)); - - // Compare with CPU. - RealScalar sol_tol = RealScalar(100) * RealScalar(n) * NumTraits<Scalar>::epsilon(); - VERIFY((x_gpu - x_cpu).norm() / (x_cpu.norm() + RealScalar(1)) < sol_tol); + check_cg_solution(A, b, x_gpu, x_cpu, tol); } -// ---- GPU CG with Jacobi preconditioner -------------------------------------- - +// ---- Extreme right-hand sides ----------------------------------------------- +// GPU analog of test_conjugate_gradient_extreme_rhs, at scales where ||b||^2 underflows to 0 or +// overflows: a nrm2 without scaling then gives rhsNorm = 0 and x = 0 reported as converged, or +// rhsNorm = inf and NaN in x. At max/4 the residual norm is within a factor 3 of max, so +// `residual /= residualScale` must divide and `x += (residualScale * alpha) * p` stay finite. template <typename Scalar> -void test_gpu_cg_jacobi(Index n) { +void test_gpu_cg_extreme_rhs() { + using SpMat = SparseMatrix<Scalar, ColMajor, int>; + using Vec = Matrix<Scalar, Dynamic, 1>; + using RealScalar = typename NumTraits<Scalar>::Real; + using Limits = std::numeric_limits<RealScalar>; + + SpMat A(2, 2); + A.insert(0, 0) = Scalar(1); + A.insert(1, 1) = Scalar(1); + A.makeCompressed(); + Vec direction(2); + direction << Scalar(1), Scalar(-1); + + gpu::Context ctx; + gpu::Context::setThreadLocal(&ctx); + gpu::SparseContext<Scalar> spmv_ctx(ctx); + auto mat = spmv_ctx.deviceView(A); + ConjugateGradient<gpu::DeviceSparseView<Scalar>, Lower | Upper, IdentityPreconditioner> cg; + cg.setTolerance(cg_tolerance<Scalar>()); + cg.compute(mat); + + const RealScalar scales[] = {numext::sqrt(Limits::denorm_min()) * RealScalar(1e-10), + numext::sqrt((Limits::max)()) * RealScalar(1e10), (Limits::max)() / RealScalar(4)}; + for (RealScalar scale : scales) { + const Vec rhs = scale * direction; + auto d_b = gpu::DeviceMatrix<Scalar>::fromHost(rhs, ctx.stream()); + for (RealScalar guess : {RealScalar(0), RealScalar(0.5)}) { + const Vec x0 = guess * rhs; + auto d_x = gpu::DeviceMatrix<Scalar>::fromHost(x0, ctx.stream()); + cg.solveWithGuessInPlace(d_b, d_x); + VERIFY_IS_EQUAL(cg.info(), Success); + VERIFY(cg.iterations() <= 1); + const Vec x = d_x.toHost(ctx.stream()); + VERIFY(x.allFinite()); + VERIFY_IS_APPROX(x / scale, direction); + } + } + gpu::Context::setThreadLocal(nullptr); +} + +// ---- The module's hand-written CG loop -------------------------------------- +// The README's loop: every scalar intermediate stays on device as a DeviceScalar and the +// convergence check is the one host sync per iteration. `jacobi` selects the diagonal +// preconditioner, applied as a cwiseProduct with 1 / diag(A). +template <typename Scalar> +void test_gpu_cg_loop(Index n, bool jacobi) { using SpMat = SparseMatrix<Scalar, ColMajor, int>; using Vec = Matrix<Scalar, Dynamic, 1>; using RealScalar = typename NumTraits<Scalar>::Real; SpMat A = make_spd<Scalar>(n); Vec b = Vec::Random(n); + const RealScalar tol = cg_tolerance<Scalar>(); + Vec x_cpu = cpu_reference(A, b, tol); - // CPU reference. - ConjugateGradient<SpMat, Lower | Upper> cpu_cg; - cpu_cg.setMaxIterations(1000); - cpu_cg.setTolerance(RealScalar(1e-8)); - cpu_cg.compute(A); - Vec x_cpu = cpu_cg.solve(b); - - // Extract inverse diagonal. Vec invdiag(n); - for (Index j = 0; j < A.outerSize(); ++j) { - typename SpMat::InnerIterator it(A, j); - while (it && it.index() != j) ++it; - if (it && it.index() == j && it.value() != Scalar(0)) - invdiag(j) = Scalar(1) / it.value(); - else - invdiag(j) = Scalar(1); - } + for (Index j = 0; j < n; ++j) invdiag(j) = Scalar(1) / A.coeff(j, j); - // GPU CG with Jacobi preconditioner. gpu::Context ctx; gpu::Context::setThreadLocal(&ctx); gpu::SparseContext<Scalar> spmv_ctx(ctx); @@ -167,16 +207,21 @@ gpu::DeviceMatrix<Scalar> d_x(n, 1); d_x.setZero(ctx); + // r = b (since x = 0) gpu::DeviceMatrix<Scalar> residual(n, 1); residual.copyFrom(ctx, d_b); RealScalar rhsNorm2 = d_b.squaredNorm(ctx); - RealScalar tol = RealScalar(1e-8); RealScalar threshold = tol * tol * rhsNorm2; RealScalar residualNorm2 = residual.squaredNorm(ctx); - // p = precond.solve(r) = invdiag .* r - gpu::DeviceMatrix<Scalar> p = d_invdiag.cwiseProduct(ctx, residual); + // p = precond.solve(r) + gpu::DeviceMatrix<Scalar> p(n, 1); + if (jacobi) { + p.cwiseProduct(ctx, d_invdiag, residual); + } else { + p.copyFrom(ctx, residual); + } gpu::DeviceMatrix<Scalar> z(n, 1), tmp(n, 1); auto absNew = residual.dot(ctx, p); @@ -192,8 +237,12 @@ residualNorm2 = residual.squaredNorm(ctx); if (residualNorm2 < threshold) break; - // z = precond.solve(r) = invdiag .* r - z.cwiseProduct(ctx, d_invdiag, residual); + // z = precond.solve(r) + if (jacobi) { + z.cwiseProduct(ctx, d_invdiag, residual); + } else { + z.copyFrom(ctx, residual); + } auto absOld = std::move(absNew); absNew = residual.dot(ctx, z); @@ -207,25 +256,23 @@ gpu::Context::setThreadLocal(nullptr); Vec x_gpu = d_x.toHost(ctx.stream()); - - Vec r = A * x_gpu - b; - RealScalar relres = r.norm() / b.norm(); - VERIFY(relres < RealScalar(1e-6)); - - RealScalar sol_tol = RealScalar(100) * RealScalar(n) * NumTraits<Scalar>::epsilon(); - VERIFY((x_gpu - x_cpu).norm() / (x_cpu.norm() + RealScalar(1)) < sol_tol); + check_cg_solution(A, b, x_gpu, x_cpu, tol); } EIGEN_DECLARE_TEST(gpu_cg) { gpu_test::require_cusparse_context(); // Split by scalar so each part compiles in parallel. - CALL_SUBTEST_1(test_gpu_cg<double>(64)); - CALL_SUBTEST_1(test_gpu_cg<double>(256)); - CALL_SUBTEST_1(test_gpu_cg_jacobi<double>(64)); - CALL_SUBTEST_1(test_gpu_cg_jacobi<double>(256)); - CALL_SUBTEST_2(test_gpu_cg<float>(64)); - CALL_SUBTEST_2(test_gpu_cg<float>(256)); - CALL_SUBTEST_2(test_gpu_cg_jacobi<float>(64)); - CALL_SUBTEST_2(test_gpu_cg_jacobi<float>(256)); + CALL_SUBTEST_1(test_gpu_cg_class<double>(64)); + CALL_SUBTEST_1(test_gpu_cg_class<double>(256)); + CALL_SUBTEST_1(test_gpu_cg_extreme_rhs<double>()); + CALL_SUBTEST_1(test_gpu_cg_loop<double>(64, false)); + CALL_SUBTEST_1(test_gpu_cg_loop<double>(256, false)); + CALL_SUBTEST_1(test_gpu_cg_loop<double>(256, true)); + CALL_SUBTEST_2(test_gpu_cg_class<float>(64)); + CALL_SUBTEST_2(test_gpu_cg_class<float>(256)); + CALL_SUBTEST_2(test_gpu_cg_extreme_rhs<float>()); + CALL_SUBTEST_2(test_gpu_cg_loop<float>(64, false)); + CALL_SUBTEST_2(test_gpu_cg_loop<float>(256, false)); + CALL_SUBTEST_2(test_gpu_cg_loop<float>(256, true)); }
diff --git a/contrib/test/GPU/cublas.cpp b/contrib/test/GPU/cublas.cpp index 3258f6c..709e98a 100644 --- a/contrib/test/GPU/cublas.cpp +++ b/contrib/test/GPU/cublas.cpp
@@ -14,6 +14,8 @@ #define EIGEN_USE_GPU #include "main.h" #include <contrib/Eigen/GPU> +#include <cstdint> +#include <new> #include "./gpu_test_helpers.h" @@ -286,6 +288,77 @@ VERIFY((C - C_ref).norm() < tol); } +// ---- GEMM with misaligned host alpha/beta ----------------------------------- +// cuBLAS reads alpha and beta as cuComplex/cuDoubleComplex, which declare 8- and +// 16-byte alignment; std::complex<double> only needs 8, so a caller may hold it +// at an odd multiple of 8. cublaslt_gemm must copy the scalars rather than pass +// such storage on: on MSVC the dispatcher's locals land there. Only the GEMV +// shapes (n == 1, or m == 1 with op(A) = A) and k == 1 fault without the copy, +// so the callers below use those. + +// The strictest alignment cuBLAS declares for a host scalar, and the offset that +// puts a scalar halfway between two such boundaries. +constexpr std::size_t kCublasScalarAlign = alignof(cuDoubleComplex); +constexpr std::size_t kMisalignment = kCublasScalarAlign / 2; + +template <typename Scalar> +void test_gemm_misaligned_host_scalars(Index m, Index n, Index k) { + using Mat = Eigen::Matrix<Scalar, Dynamic, Dynamic>; + using RealScalar = typename NumTraits<Scalar>::Real; + + Mat A = Mat::Random(m, k); + Mat B = Mat::Random(k, n); + + gpu::Context ctx; + auto d_A = gpu::DeviceMatrix<Scalar>::fromHost(A, ctx.stream()); + auto d_B = gpu::DeviceMatrix<Scalar>::fromHost(B, ctx.stream()); + gpu::DeviceMatrix<Scalar> d_C(m, n); + + alignas(kCublasScalarAlign) unsigned char storage[kMisalignment + 2 * sizeof(Scalar)]; + const Scalar* alpha = ::new (storage + kMisalignment) Scalar(2); + const Scalar* beta = ::new (storage + kMisalignment + sizeof(Scalar)) Scalar(0); + VERIFY_IS_EQUAL(reinterpret_cast<std::uintptr_t>(alpha) % kCublasScalarAlign, std::uintptr_t(kMisalignment)); + + gpu::internal::cublaslt_gemm(ctx.cublasLtHandle(), ctx.cublasHandle(), CUBLAS_OP_N, CUBLAS_OP_N, m, n, k, alpha, + d_A.data(), m, d_B.data(), k, beta, d_C.data(), m, ctx.gemmWorkspace(), + ctx.gemmPlanCache(), ctx.cublasLtMaxWorkspaceBytes(), ctx.stream()); + d_C.recordReady(ctx.stream()); + + Mat C = d_C.toHost(); + Mat C_ref = Scalar(2) * A * B; + + RealScalar tol = RealScalar(k) * NumTraits<Scalar>::epsilon() * C_ref.norm(); + VERIFY((C - C_ref).norm() < tol); +} + +// ---- BLAS-1 with misaligned host alpha -------------------------------------- +// Zaxpy and Zscal fault on a host-mode alpha at 8 mod 16 like the GEMV paths +// above; the complex axpy/scal wrappers must copy it. + +template <typename Scalar> +void test_blas1_misaligned_host_alpha(Index n) { + using Vec = Eigen::Matrix<Scalar, Dynamic, 1>; + + Vec x = Vec::Random(n); + Vec y = Vec::Random(n); + + gpu::Context ctx; + auto d_x = gpu::DeviceMatrix<Scalar>::fromHost(x, ctx.stream()); + auto d_y = gpu::DeviceMatrix<Scalar>::fromHost(y, ctx.stream()); + + alignas(kCublasScalarAlign) unsigned char storage[kMisalignment + sizeof(Scalar)]; + const Scalar* alpha = ::new (storage + kMisalignment) Scalar(2); + VERIFY_IS_EQUAL(reinterpret_cast<std::uintptr_t>(alpha) % kCublasScalarAlign, std::uintptr_t(kMisalignment)); + + EIGEN_CUBLAS_CHECK(gpu::internal::cublasXaxpy(ctx.cublasHandle(), n, alpha, d_x.data(), 1, d_y.data(), 1)); + EIGEN_CUBLAS_CHECK(gpu::internal::cublasXscal(ctx.cublasHandle(), n, alpha, d_x.data(), 1)); + d_x.recordReady(ctx.stream()); + d_y.recordReady(ctx.stream()); + + VERIFY_IS_APPROX(Vec(d_y.toHost()), Vec(y + Scalar(2) * x)); + VERIFY_IS_APPROX(Vec(d_x.toHost()), Vec(Scalar(2) * x)); +} + // ---- GEMM cross-context reuse of the same destination ----------------------- template <typename Scalar> @@ -373,6 +446,46 @@ VERIFY((D - D_ref).norm() < tol); } +// ---- GEMM with m, n, or k == 0 --------------------------------------------- + +template <typename Scalar> +void test_gemm_empty_dim(Index m, Index n, Index k) { + using Mat = Eigen::Matrix<Scalar, Dynamic, Dynamic>; + + Mat A = Mat::Random(m, k); + Mat B = Mat::Random(k, n); + Mat C_init = Mat::Random(m, n); + const Mat zero = Mat::Zero(m, n); + + auto d_A = gpu::DeviceMatrix<Scalar>::fromHost(A); + auto d_B = gpu::DeviceMatrix<Scalar>::fromHost(B); + auto d_AH = gpu::DeviceMatrix<Scalar>::fromHost(Mat(A.adjoint())); + auto d_BT = gpu::DeviceMatrix<Scalar>::fromHost(Mat(B.transpose())); + + // A * B is the m x n zero matrix: `=` overwrites a stale destination, and + // `+=` / `-=` leave C unchanged. + auto d_C = gpu::DeviceMatrix<Scalar>::fromHost(C_init); + d_C = d_A * d_B; + VERIFY_IS_CWISE_EQUAL(d_C.toHost(), zero); + + gpu::DeviceMatrix<Scalar> d_D; + d_D = d_AH.adjoint() * d_BT.transpose(); + VERIFY_IS_CWISE_EQUAL(d_D.toHost(), zero); + + d_C = gpu::DeviceMatrix<Scalar>::fromHost(C_init); + d_C += d_A * d_BT.transpose(); + VERIFY_IS_CWISE_EQUAL(d_C.toHost(), C_init); + + gpu::Context ctx; + d_C.device(ctx) -= (Scalar(2) * d_AH.adjoint()) * d_B; + VERIFY_IS_CWISE_EQUAL(d_C.toHost(), C_init); + + // An empty destination accumulates from zero. + gpu::DeviceMatrix<Scalar> d_E; + d_E += d_A * d_B; + VERIFY_IS_CWISE_EQUAL(d_E.toHost(), zero); +} + // ---- LLT solve expression: d_X = d_A.llt().solve(d_B) ---------------------- template <typename MatrixType> @@ -740,9 +853,15 @@ CALL_SUBTEST(test_gemm_subtract<Scalar>(64, 64, 64)); CALL_SUBTEST(test_gemm_subtract_empty<Scalar>(64, 64, 64)); CALL_SUBTEST(test_gemm_explicit_context<Scalar>(64, 64, 64)); + CALL_SUBTEST(test_gemm_misaligned_host_scalars<Scalar>(64, 1, 32)); + CALL_SUBTEST(test_gemm_misaligned_host_scalars<Scalar>(1, 48, 32)); + CALL_SUBTEST(test_blas1_misaligned_host_alpha<Scalar>(1000)); CALL_SUBTEST(test_gemm_cross_context_reuse<Scalar>(64)); CALL_SUBTEST(test_gemm_cross_context_resize<Scalar>()); CALL_SUBTEST(test_gemm_chain<Scalar>(64)); + CALL_SUBTEST(test_gemm_empty_dim<Scalar>(0, 5, 3)); + CALL_SUBTEST(test_gemm_empty_dim<Scalar>(4, 0, 3)); + CALL_SUBTEST(test_gemm_empty_dim<Scalar>(4, 5, 0)); // Solver expressions — zero-size edge cases (use dedicated tests, not residual-based) @@ -788,6 +907,7 @@ // Negative definite matrix — LLT factorization must fail. MatrixXd A = -MatrixXd::Identity(8, 8); gpu::LLT<double> llt(A); + if (gpu_test::cusolver_potrf_missed_non_spd(llt.info())) return; VERIFY_IS_EQUAL(llt.info(), NumericalIssue); }
diff --git a/contrib/test/GPU/cusolver_eigen.cpp b/contrib/test/GPU/cusolver_eigen.cpp index c93f3e9..40b7a75 100644 --- a/contrib/test/GPU/cusolver_eigen.cpp +++ b/contrib/test/GPU/cusolver_eigen.cpp
@@ -118,6 +118,16 @@ Mat A_hat = V * W_gpu.asDiagonal() * V.adjoint(); RealScalar tol = RealScalar(8) * static_cast<RealScalar>(n) * NumTraits<Scalar>::epsilon() * A.norm(); VERIFY((A_hat - A).norm() < tol); + + // Context-bound rvalue constructor adopts d_A instead of copying it. + gpu::Context ctx; + auto d_A_ctx = gpu::DeviceMatrix<Scalar>::fromHost(A, ctx.stream()); + gpu::SelfAdjointEigenSolver<Scalar> es_ctx(ctx, std::move(d_A_ctx)); + VERIFY(d_A_ctx.empty()); + VERIFY_IS_EQUAL(es_ctx.info(), Success); + VERIFY(es_ctx.stream() == ctx.stream()); + Mat V_ctx = es_ctx.eigenvectors(); + VERIFY((V_ctx * es_ctx.eigenvalues().asDiagonal() * V_ctx.adjoint() - A).norm() < tol); } // ---- Recompute (reuse solver object) ---------------------------------------- @@ -287,6 +297,26 @@ VERIFY_IS_EQUAL(context_solver.info(), Success); } +// ---- Host getters wait only for the solver's stream ------------------------- +// A download through the legacy default stream (cudaMemcpy) would also wait +// for the unrelated blocking stream parked below. + +template <typename Scalar> +void test_eigen_getters_ignore_other_streams(Index n) { + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + + Mat R = Mat::Random(n, n); + Mat A = R + R.adjoint(); + gpu::SelfAdjointEigenSolver<Scalar> es(A); + VERIFY_IS_EQUAL(es.info(), Success); + + gpu::Context other; + gpu_test::ParkedStream parked(other.stream()); + (void)es.eigenvalues(); + (void)es.eigenvectors(); + VERIFY(parked.held()); +} + // ---- Empty matrix ----------------------------------------------------------- void test_eigen_empty() { @@ -329,6 +359,8 @@ // Move constructor/assignment. CALL_SUBTEST(test_eigen_move<Scalar>(32)); + + CALL_SUBTEST(test_eigen_getters_ignore_other_streams<Scalar>(64)); } EIGEN_DECLARE_TEST(gpu_cusolver_eigen) {
diff --git a/contrib/test/GPU/cusolver_llt.cpp b/contrib/test/GPU/cusolver_llt.cpp index f70b6db..ea87072 100644 --- a/contrib/test/GPU/cusolver_llt.cpp +++ b/contrib/test/GPU/cusolver_llt.cpp
@@ -119,6 +119,7 @@ void test_not_spd() { Eigen::MatrixXd h_A = -Eigen::MatrixXd::Identity(8, 8); // negative definite Eigen::gpu::LLT<double> gpu_llt(h_A); + if (gpu_test::cusolver_potrf_missed_non_spd(gpu_llt.info())) return; VERIFY_IS_EQUAL(gpu_llt.info(), Eigen::NumericalIssue); } @@ -128,6 +129,7 @@ Eigen::MatrixXd h_A = -Eigen::MatrixXd::Identity(8, 8); Eigen::MatrixXd h_B = Eigen::MatrixXd::Random(8, 4); Eigen::gpu::LLT<double> gpu_llt(h_A); + if (gpu_test::cusolver_potrf_missed_non_spd(gpu_llt.info())) return; VERIFY_IS_EQUAL(gpu_llt.info(), Eigen::NumericalIssue); auto d_B = Eigen::gpu::DeviceMatrix<double>::fromHost(h_B); VERIFY_RAISES_ASSERT(gpu_llt.solve(d_B)); @@ -243,6 +245,14 @@ Mat X2 = d_X2.toHost(); VERIFY((A * X2 - B).norm() / B.norm() < RealScalar(n) * NumTraits<Scalar>::epsilon()); VERIFY(d_B.data() == nullptr); // moved-from + + // Context-bound rvalue constructor adopts d_A instead of copying it. + gpu::LLT<Scalar> llt_adopt(ctx, std::move(d_A)); + VERIFY(d_A.empty()); + VERIFY(llt_adopt.info() == Success); + VERIFY(llt_adopt.stream() == ctx.stream()); + Mat X3 = llt_adopt.solve(B); + VERIFY((A * X3 - B).norm() / B.norm() < RealScalar(n) * NumTraits<Scalar>::epsilon()); } // ---- Non-plain host input ---------------------------------------------------
diff --git a/contrib/test/GPU/cusolver_lu.cpp b/contrib/test/GPU/cusolver_lu.cpp index 8c0b776..c80379a 100644 --- a/contrib/test/GPU/cusolver_lu.cpp +++ b/contrib/test/GPU/cusolver_lu.cpp
@@ -118,6 +118,17 @@ MatrixType X = lu.solve(B); RealScalar residual = (A * X - B).norm() / (A.norm() * X.norm()); VERIFY(residual < RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon()); + + // Context-bound rvalue constructor adopts as well. + gpu::Context ctx; + auto d_A_ctx = gpu::DeviceMatrix<Scalar>::fromHost(A, ctx.stream()); + gpu::LU<Scalar> lu_ctx(ctx, std::move(d_A_ctx)); + VERIFY(d_A_ctx.empty()); + VERIFY_IS_EQUAL(lu_ctx.info(), Success); + VERIFY(lu_ctx.stream() == ctx.stream()); + MatrixType X_ctx = lu_ctx.solve(B); + residual = (A * X_ctx - B).norm() / (A.norm() * X_ctx.norm()); + VERIFY(residual < RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon()); } template <typename Scalar>
diff --git a/contrib/test/GPU/cusolver_qr.cpp b/contrib/test/GPU/cusolver_qr.cpp index c1baaa6..dc9764a 100644 --- a/contrib/test/GPU/cusolver_qr.cpp +++ b/contrib/test/GPU/cusolver_qr.cpp
@@ -92,6 +92,17 @@ RealScalar residual = (A * X - B).norm() / (A.norm() * X.norm()); VERIFY(residual < RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon()); + + // Context-bound rvalue constructor adopts a square A instead of copying it. + gpu::Context ctx; + auto d_A_ctx = gpu::DeviceMatrix<Scalar>::fromHost(A, ctx.stream()); + gpu::QR<Scalar> qr_ctx(ctx, std::move(d_A_ctx)); + VERIFY(d_A_ctx.empty()); + VERIFY_IS_EQUAL(qr_ctx.info(), Success); + VERIFY(qr_ctx.stream() == ctx.stream()); + Mat X_ctx = qr_ctx.solve(B); + residual = (A * X_ctx - B).norm() / (A.norm() * X_ctx.norm()); + VERIFY(residual < RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon()); } // ---- Solve overdetermined via device path ----------------------------------- @@ -216,6 +227,24 @@ } } +// ---- matrixR() waits only for the solver's stream --------------------------- +// A download through the legacy default stream (cudaMemcpy) would also wait +// for the unrelated blocking stream parked below. + +template <typename Scalar> +void test_qr_matrixR_ignores_other_streams(Index m, Index n) { + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + + Mat A = Mat::Random(m, n); + gpu::QR<Scalar> qr(A); + VERIFY_IS_EQUAL(qr.info(), Success); + + gpu::Context other; + gpu_test::ParkedStream parked(other.stream()); + (void)qr.matrixR(); + VERIFY(parked.held()); +} + // ---- Multiple solves reuse the factorization -------------------------------- template <typename Scalar> @@ -254,6 +283,7 @@ CALL_SUBTEST(test_qr_matrixR<Scalar>(64, 64)); CALL_SUBTEST(test_qr_matrixR<Scalar>(128, 64)); + CALL_SUBTEST(test_qr_matrixR_ignores_other_streams<Scalar>(96, 64)); CALL_SUBTEST(test_qr_solve_device<Scalar>(64, 4)); CALL_SUBTEST(test_qr_solve_overdetermined_device<Scalar>(128, 64, 4));
diff --git a/contrib/test/GPU/cusolver_svd.cpp b/contrib/test/GPU/cusolver_svd.cpp index 3b34a03..fb890d0 100644 --- a/contrib/test/GPU/cusolver_svd.cpp +++ b/contrib/test/GPU/cusolver_svd.cpp
@@ -137,6 +137,16 @@ const RealScalar cond = S(0) / S(S.size() - 1); const RealScalar tol = RealScalar(8) * cond * NumTraits<Scalar>::epsilon(); VERIFY((X - X_ref).norm() / X_ref.norm() < tol); + + // Context-bound rvalue constructor adopts A when m >= n (a wide A is + // transposed into a copy). + auto d_A = gpu::DeviceMatrix<Scalar>::fromHost(A, ctx.stream()); + gpu::SVD<Scalar> svd_adopt(ctx, std::move(d_A), ComputeThinU | ComputeThinV); + VERIFY(m < n || d_A.empty()); + VERIFY_IS_EQUAL(svd_adopt.info(), Success); + VERIFY(svd_adopt.stream() == ctx.stream()); + Mat X_adopt = svd_adopt.solve(B); + VERIFY((X_adopt - X_ref).norm() / X_ref.norm() < tol); } // ---- Solve: truncated ------------------------------------------------------- @@ -384,6 +394,26 @@ VERIFY_IS_EQUAL(VT_again.cols(), n); } +// ---- Host getters wait only for the solver's stream ------------------------- +// A download through the legacy default stream (cudaMemcpy) would also wait +// for the unrelated blocking stream parked below. + +template <typename Scalar> +void test_svd_getters_ignore_other_streams(Index m, Index n) { + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + + Mat A = Mat::Random(m, n); + gpu::SVD<Scalar> svd(A, ComputeThinU | ComputeThinV); + VERIFY_IS_EQUAL(svd.info(), Success); + + gpu::Context other; + gpu_test::ParkedStream parked(other.stream()); + (void)svd.singularValues(); + (void)svd.matrixU(); + (void)svd.matrixVT(); + VERIFY(parked.held()); +} + // ---- Empty matrix ----------------------------------------------------------- void test_svd_empty() { @@ -481,6 +511,9 @@ // Chain device views into a downstream GEMM (orthogonality check). CALL_SUBTEST(test_svd_chain_orthogonality<Scalar>(64, 64)); CALL_SUBTEST(test_svd_chain_orthogonality<Scalar>(96, 64)); + + CALL_SUBTEST(test_svd_getters_ignore_other_streams<Scalar>(64, 48)); + CALL_SUBTEST(test_svd_getters_ignore_other_streams<Scalar>(48, 64)); // wide: U and V^T swap roles } EIGEN_DECLARE_TEST(gpu_cusolver_svd) {
diff --git a/contrib/test/GPU/cusparse_spmv.cpp b/contrib/test/GPU/cusparse_spmv.cpp index ef62165..f2c11ce 100644 --- a/contrib/test/GPU/cusparse_spmv.cpp +++ b/contrib/test/GPU/cusparse_spmv.cpp
@@ -321,6 +321,98 @@ VERIFY((tmp_gpu - y_cpu).norm() / (y_cpu.norm() + RealScalar(1)) < tol); } +// ---- Expression syntax with a dense addend: d_y = d_b ± d_A * d_x ----------- + +template <typename Scalar> +void test_spmv_affine_expr(Index n) { + using SpMat = SparseMatrix<Scalar, ColMajor, int>; + using Vec = Matrix<Scalar, Dynamic, 1>; + using RealScalar = typename NumTraits<Scalar>::Real; + + SpMat A = make_sparse<Scalar>(n, n); + Vec x = Vec::Random(n), b = Vec::Random(n); + const Vec Ax = A * x; + + // The addend copy runs on the thread-local Context and the SpMV on the SparseContext's + // stream. A second Context makes those two different streams; the readiness events + // must order the copy before the product. + gpu::Context gpu_ctx, copy_ctx; + gpu::Context::setThreadLocal(©_ctx); + gpu::SparseContext<Scalar> ctx(gpu_ctx); + auto d_A = ctx.deviceView(A); + auto d_x = gpu::DeviceMatrix<Scalar>::fromHost(x, gpu_ctx.stream()); + auto d_b = gpu::DeviceMatrix<Scalar>::fromHost(b, gpu_ctx.stream()); + + const RealScalar tol = RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon(); + auto check = [&](const gpu::DeviceMatrix<Scalar>& d_y, const Vec& y_ref) { + Vec y = d_y.toHost(gpu_ctx.stream()); + VERIFY((y - y_ref).norm() / (y_ref.norm() + RealScalar(1)) < tol); + }; + + gpu::DeviceMatrix<Scalar> d_y = d_b - d_A * d_x; // copy-initialization + check(d_y, b - Ax); + d_y = d_b + d_A * d_x; + check(d_y, b + Ax); + d_y = d_A * d_x + d_b; + check(d_y, Ax + b); + d_y.noalias() = d_A * d_x - d_b; + check(d_y, Ax - b); + + // The addend is the destination: beta accumulates in place, without a copy. + gpu::DeviceMatrix<Scalar> d_r = d_b; + d_r = d_r - d_A * d_x; + check(d_r, b - Ax); + gpu::Context::setThreadLocal(nullptr); +} + +template <typename Scalar> +void test_device_spmm_affine(Index n, Index nrhs) { + using SpMat = SparseMatrix<Scalar, ColMajor, int>; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using RealScalar = typename NumTraits<Scalar>::Real; + + SpMat A = make_sparse<Scalar>(n, n); + Mat X = Mat::Random(n, nrhs), B = Mat::Random(n, nrhs); + + gpu::Context gctx; + gpu::Context::setThreadLocal(&gctx); + gpu::SparseContext<Scalar> ctx(gctx); + auto view = ctx.deviceView(A); + auto d_X = gpu::DeviceMatrix<Scalar>::fromHost(X, gctx.stream()); + auto d_B = gpu::DeviceMatrix<Scalar>::fromHost(B, gctx.stream()); + gpu::DeviceMatrix<Scalar> d_Y = d_B - view * d_X; // nrhs > 1 -> SpMM with beta = 1 + gpu::Context::setThreadLocal(nullptr); + + Mat Y_ref = B - A * X; + RealScalar tol = RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon(); + VERIFY((d_Y.toHost() - Y_ref).norm() / (Y_ref.norm() + RealScalar(1)) < tol); +} + +// A matrix without stored entries reduces d_b ± d_A * d_x to ±d_b. +template <typename Scalar> +void test_affine_empty(Index n) { + using SpMat = SparseMatrix<Scalar, ColMajor, int>; + using Vec = Matrix<Scalar, Dynamic, 1>; + + SpMat A(n, n); + A.makeCompressed(); + Vec x = Vec::Random(n), b = Vec::Random(n); + + gpu::Context gctx; + gpu::Context::setThreadLocal(&gctx); + gpu::SparseContext<Scalar> ctx(gctx); + auto d_A = ctx.deviceView(A); + VERIFY_IS_EQUAL(d_A.nonZeros(), 0); + auto d_x = gpu::DeviceMatrix<Scalar>::fromHost(x, gctx.stream()); + auto d_b = gpu::DeviceMatrix<Scalar>::fromHost(b, gctx.stream()); + + gpu::DeviceMatrix<Scalar> d_y = d_b - d_A * d_x; + VERIFY(d_y.toHost(gctx.stream()) == b); + d_y = d_A * d_x - d_b; + VERIFY(d_y.toHost(gctx.stream()) == -b); + gpu::Context::setThreadLocal(nullptr); +} + // ---- deviceView overwrite: second view replaces first ----------------------- template <typename Scalar> @@ -497,6 +589,9 @@ CALL_SUBTEST(test_empty<Scalar>()); CALL_SUBTEST(test_spmv_device<Scalar>(64)); CALL_SUBTEST(test_spmv_expr<Scalar>(64)); + CALL_SUBTEST(test_spmv_affine_expr<Scalar>(64)); + CALL_SUBTEST(test_device_spmm_affine<Scalar>(64, 5)); + CALL_SUBTEST(test_affine_empty<Scalar>(16)); CALL_SUBTEST(test_deviceview_overwrite<Scalar>(64)); }
diff --git a/contrib/test/GPU/device_matrix.cpp b/contrib/test/GPU/device_matrix.cpp index 6be1af7..51dab01 100644 --- a/contrib/test/GPU/device_matrix.cpp +++ b/contrib/test/GPU/device_matrix.cpp
@@ -352,6 +352,72 @@ VERIFY_IS_EQUAL(result.cols(), 0); } +// ---- isView() --------------------------------------------------------------- + +void test_is_view() { + const MatrixXd A = MatrixXd::Random(8, 4); + auto owner = gpu::DeviceMatrix<double>::fromHost(A); + VERIFY(!owner.isView()); + VERIFY(!gpu::DeviceMatrix<double>().isView()); + + auto v = gpu::DeviceMatrix<double>::view(owner.data(), 8, 4); + VERIFY(v.isView()); + VERIFY_IS_EQUAL(v.toHost(), A); + gpu::DeviceMatrix<double> moved(std::move(v)); + VERIFY(moved.isView()); + + // release() hands back the owner's pointer, frees nothing, and leaves no view. + VERIFY(moved.release() == owner.data()); + VERIFY(!moved.isView()); + VERIFY_IS_EQUAL(owner.toHost(), A); + + // A new size gives the view storage of its own. + auto w = gpu::DeviceMatrix<double>::view(owner.data(), 8, 4); + w.resize(2, 2); + VERIFY(!w.isView()); + VERIFY(w.data() != owner.data()); + auto z = gpu::DeviceMatrix<double>::view(owner.data(), 8, 4); + z.resize(0, 0); + VERIFY(!z.isView()); + VERIFY_IS_EQUAL(owner.toHost(), A); +} + +// ---- Moving a view into a solver -------------------------------------------- + +// A view borrows another object's storage, so a solver that takes a matrix by +// move must copy a view rather than factor over (and later free) that storage. +void test_solver_copies_moved_view() { + const Index n = 32; + const MatrixXd A = MatrixXd::Random(n, n) + MatrixXd::Identity(n, n) * double(n); + const MatrixXd B = MatrixXd::Random(n, 2); + gpu::SVD<double> svd(A); + const MatrixXd U = svd.matrixU(); + gpu::LU<double> lu; + lu.compute(svd.d_matrixU()); + VERIFY_IS_APPROX(svd.matrixU(), U); + VERIFY_IS_APPROX(MatrixXd(U * lu.solve(B)), B); + const MatrixXd X = lu.solve(svd.d_matrixU()).toHost(); + VERIFY_IS_APPROX(svd.matrixU(), U); + VERIFY_IS_APPROX(MatrixXd(U * X), U); + gpu::Context ctx; + gpu::LU<double> lu_ctx(ctx, svd.d_matrixU()); + VERIFY_IS_APPROX(svd.matrixU(), U); + VERIFY_IS_APPROX(MatrixXd(U * lu_ctx.solve(B)), B); + + // The remaining adopting overloads, each given a view of d_S. + const MatrixXd S = A * A.transpose(); + auto d_S = gpu::DeviceMatrix<double>::fromHost(S); + const auto view_of_S = [&] { return gpu::DeviceMatrix<double>::view(d_S.data(), n, n); }; + gpu::LLT<double> llt(view_of_S()); + VERIFY_IS_APPROX(MatrixXd(S * llt.solve(view_of_S()).toHost()), S); + gpu::QR<double> qr(view_of_S()); + VERIFY_IS_APPROX(MatrixXd(S * qr.solve(B)), B); + gpu::SVD<double> svd_of_S(view_of_S()); + gpu::SelfAdjointEigenSolver<double> es(view_of_S()); + VERIFY_IS_APPROX(VectorXd(svd_of_S.singularValues().reverse()), es.eigenvalues()); + VERIFY_IS_EQUAL(d_S.toHost(), S); +} + // ---- Per-scalar driver ------------------------------------------------------ template <typename Scalar> @@ -545,6 +611,47 @@ } } +// ---- BLAS-1 scaled by a gpu::DeviceScalar (device pointer mode) ------------- + +template <typename Scalar> +void test_device_scalar_blas1() { + using Vec = Matrix<Scalar, Dynamic, 1>; + using RealScalar = typename NumTraits<Scalar>::Real; + + const Index n = 256; + const Vec x = Vec::Random(n); + const Vec y = Vec::Random(n); + const Scalar alpha = gpu_test::make_test_value<Scalar>(RealScalar(1.5), RealScalar(-0.75)); + // |fl(x + a*y) - (x + a*y)| <= (1/2 + sqrt(2)) eps (|x| + |a| |y|) elementwise, for the device result and again + // for the host reference. + const RealScalar tol = RealScalar(4) * NumTraits<Scalar>::epsilon() * (x.norm() + numext::abs(alpha) * y.norm()); + + gpu::Context ctx; + gpu::Context::setThreadLocal(&ctx); + const gpu::DeviceScalar<Scalar> d_alpha(alpha, ctx.stream()); + + // x *= d_alpha (scal) + { + const Vec ref = alpha * x; + auto d_x = gpu::DeviceMatrix<Scalar>::fromHost(x, ctx.stream()); + d_x *= d_alpha; + const Vec result = d_x.toHost(ctx.stream()); + VERIFY((result - ref).norm() <= tol); + } + + // x += d_alpha * y (axpy) + { + const Vec ref = x + alpha * y; + auto d_x = gpu::DeviceMatrix<Scalar>::fromHost(x, ctx.stream()); + auto d_y = gpu::DeviceMatrix<Scalar>::fromHost(y, ctx.stream()); + d_x += d_alpha * d_y; + const Vec result = d_x.toHost(ctx.stream()); + VERIFY((result - ref).norm() <= tol); + } + + gpu::Context::setThreadLocal(nullptr); +} + // ---- cwiseProduct ----------------------------------------------------------- template <typename Scalar> @@ -567,6 +674,111 @@ VERIFY((result - ref).norm() < tol * ref.norm() + tol); } +// ---- Reductions into a caller's DeviceScalar ---------------------------------- + +template <typename Scalar> +void test_reductions_into() { + using Vec = Matrix<Scalar, Dynamic, 1>; + using RealScalar = typename NumTraits<Scalar>::Real; + const Index n = 300; + const RealScalar tol = RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon(); + const Vec a = Vec::Random(n), b = Vec::Random(n); + + gpu::Context ctx; + auto d_a = gpu::DeviceMatrix<Scalar>::fromHost(a, ctx.stream()); + auto d_b = gpu::DeviceMatrix<Scalar>::fromHost(b, ctx.stream()); + gpu::DeviceScalar<Scalar> dot(ctx.stream()); + gpu::DeviceScalar<RealScalar> squared_norm(ctx.stream()), norm(ctx.stream()), stable_norm(ctx.stream()); + const Scalar* dot_slot = dot.devicePtr(); + const RealScalar* norm_slot = norm.devicePtr(); + + // Repeated reductions write the same storage. + for (int rep = 0; rep < 2; ++rep) { + d_a.dot(ctx, d_b, dot); + d_a.squaredNorm(ctx, squared_norm); + d_a.norm(ctx, norm); + d_a.stableNorm(ctx, stable_norm); + VERIFY(dot.devicePtr() == dot_slot); + VERIFY(norm.devicePtr() == norm_slot); + VERIFY(numext::abs(Scalar(dot) - a.dot(b)) < tol * numext::abs(a.dot(b)) + tol); + VERIFY(numext::abs(RealScalar(squared_norm) - a.squaredNorm()) < tol * a.squaredNorm()); + VERIFY(numext::abs(RealScalar(norm) - a.norm()) < tol * a.norm()); + VERIFY(numext::abs(RealScalar(stable_norm) - a.norm()) < tol * a.norm()); + } + + // stableNorm() scales its accumulation: exact where x^H x overflows. + const RealScalar big = numext::sqrt((NumTraits<RealScalar>::highest)()) * RealScalar(4); + auto d_big = gpu::DeviceMatrix<Scalar>::fromHost(Vec::Constant(n, Scalar(big)), ctx.stream()); + d_big.stableNorm(ctx, stable_norm); + VERIFY_IS_APPROX(RealScalar(stable_norm), big * numext::sqrt(RealScalar(n))); + + // An empty matrix writes zero over the previous result. + gpu::DeviceMatrix<Scalar> d_empty(0, 1); + d_empty.dot(ctx, d_empty, dot); + d_empty.squaredNorm(ctx, squared_norm); + d_empty.norm(ctx, norm); + VERIFY_IS_EQUAL(Scalar(dot), Scalar(0)); + VERIFY_IS_EQUAL(RealScalar(squared_norm), RealScalar(0)); + VERIFY_IS_EQUAL(RealScalar(norm), RealScalar(0)); +} + +// Reductions into existing DeviceScalars enqueue no allocation and no host +// sync -- for complex scalars too, whose squaredNorm() used to read the complex +// dot product back to take its real part -- so they capture into a CUDA graph +// with no memory nodes, and the graph replays them. +template <typename Scalar> +void test_reductions_capture() { + using Vec = Matrix<Scalar, Dynamic, 1>; + using RealScalar = typename NumTraits<Scalar>::Real; + const Index n = 128; + const RealScalar tol = RealScalar(10) * RealScalar(n) * NumTraits<Scalar>::epsilon(); + const Vec a = Vec::Random(n); + + gpu::Context ctx; + auto d_a = gpu::DeviceMatrix<Scalar>::fromHost(a, ctx.stream()); + gpu::DeviceScalar<Scalar> dot(ctx.stream()); + gpu::DeviceScalar<RealScalar> squared_norm(ctx.stream()), norm(ctx.stream()); + // The first calls create the cuBLAS handle and load the kernels, which a + // capture would reject. + d_a.dot(ctx, d_a, dot); + d_a.squaredNorm(ctx, squared_norm); + d_a.norm(ctx, norm); + EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(ctx.stream())); + + cudaGraph_t graph = nullptr; + EIGEN_CUDA_RUNTIME_CHECK(cudaStreamBeginCapture(ctx.stream(), cudaStreamCaptureModeRelaxed)); + d_a.dot(ctx, d_a, dot); + d_a.squaredNorm(ctx, squared_norm); + d_a.norm(ctx, norm); + VERIFY(cudaStreamEndCapture(ctx.stream(), &graph) == cudaSuccess); + + size_t count = 0; + EIGEN_CUDA_RUNTIME_CHECK(cudaGraphGetNodes(graph, nullptr, &count)); + std::vector<cudaGraphNode_t> nodes(count); + EIGEN_CUDA_RUNTIME_CHECK(cudaGraphGetNodes(graph, nodes.data(), &count)); + VERIFY(count > 0); + for (cudaGraphNode_t node : nodes) { + cudaGraphNodeType type; + EIGEN_CUDA_RUNTIME_CHECK(cudaGraphNodeGetType(node, &type)); + VERIFY(type != cudaGraphNodeTypeMemAlloc && type != cudaGraphNodeTypeMemFree); + } + + cudaGraphExec_t exec = nullptr; + // cudaGraphInstantiate took five arguments before CUDA 12. + EIGEN_CUDA_RUNTIME_CHECK(cudaGraphInstantiateWithFlags(&exec, graph, 0)); + // Clear the slots so that the checks below see the replay's results. + EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(dot.devicePtr(), 0, sizeof(Scalar), ctx.stream())); + EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(squared_norm.devicePtr(), 0, sizeof(RealScalar), ctx.stream())); + EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(norm.devicePtr(), 0, sizeof(RealScalar), ctx.stream())); + EIGEN_CUDA_RUNTIME_CHECK(cudaGraphLaunch(exec, ctx.stream())); + EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(ctx.stream())); + VERIFY(numext::abs(Scalar(dot) - a.squaredNorm()) < tol * a.squaredNorm()); + VERIFY(numext::abs(RealScalar(squared_norm) - a.squaredNorm()) < tol * a.squaredNorm()); + VERIFY(numext::abs(RealScalar(norm) - a.norm()) < tol * a.norm()); + EIGEN_CUDA_RUNTIME_CHECK(cudaGraphExecDestroy(exec)); + EIGEN_CUDA_RUNTIME_CHECK(cudaGraphDestroy(graph)); +} + EIGEN_DECLARE_TEST(gpu_device_matrix) { gpu_test::require_cuda_device(); CALL_SUBTEST(test_default_construct()); @@ -575,6 +787,8 @@ CALL_SUBTEST(test_host_transfer_ready()); CALL_SUBTEST(test_host_transfer_move()); CALL_SUBTEST(test_host_transfer_move_assign()); + CALL_SUBTEST(test_is_view()); + CALL_SUBTEST(test_solver_copies_moved_view()); CALL_SUBTEST((test_allocate<float>(100, 50))); CALL_SUBTEST((test_allocate<double>(100, 50))); CALL_SUBTEST(test_scalar<float>()); @@ -593,6 +807,16 @@ CALL_SUBTEST(test_device_scalar<double>()); CALL_SUBTEST(test_device_scalar<std::complex<float>>()); CALL_SUBTEST(test_device_scalar<std::complex<double>>()); + CALL_SUBTEST(test_device_scalar_blas1<float>()); + CALL_SUBTEST(test_device_scalar_blas1<double>()); + CALL_SUBTEST(test_device_scalar_blas1<std::complex<float>>()); + CALL_SUBTEST(test_device_scalar_blas1<std::complex<double>>()); CALL_SUBTEST(test_cwiseProduct<float>()); CALL_SUBTEST(test_cwiseProduct<double>()); + CALL_SUBTEST(test_reductions_into<float>()); + CALL_SUBTEST(test_reductions_into<double>()); + CALL_SUBTEST(test_reductions_into<std::complex<float>>()); + CALL_SUBTEST(test_reductions_into<std::complex<double>>()); + CALL_SUBTEST(test_reductions_capture<double>()); + CALL_SUBTEST(test_reductions_capture<std::complex<float>>()); }
diff --git a/contrib/test/GPU/gpu_test_helpers.h b/contrib/test/GPU/gpu_test_helpers.h index 968cddc..7801c41 100644 --- a/contrib/test/GPU/gpu_test_helpers.h +++ b/contrib/test/GPU/gpu_test_helpers.h
@@ -18,8 +18,12 @@ #define EIGEN_UNSUPPORTED_TEST_GPU_TEST_HELPERS_H #include <Eigen/Core> +#include <atomic> +#include <chrono> +#include <condition_variable> #include <cstdlib> #include <iostream> +#include <mutex> #include <type_traits> namespace gpu_test { @@ -48,6 +52,68 @@ std::exit(77); } } + +// Parks `stream` behind a host function that returns when the ParkedStream is +// destroyed or after `timeout`, whichever comes first. An operation that does +// not wait for `stream` returns while held() is still true; one that does +// returns only after the timeout, when held() is false. +class ParkedStream { + public: + explicit ParkedStream(cudaStream_t stream, std::chrono::milliseconds timeout = std::chrono::seconds(1)) + : stream_(stream), timeout_(timeout) { + EIGEN_CUDA_RUNTIME_CHECK(cudaLaunchHostFunc(stream_, &ParkedStream::hold, this)); + } + + ~ParkedStream() { + { + std::lock_guard<std::mutex> lock(mutex_); + unpark_ = true; + } + cv_.notify_one(); + (void)cudaStreamSynchronize(stream_); + } + + ParkedStream(const ParkedStream&) = delete; + ParkedStream& operator=(const ParkedStream&) = delete; + + bool held() const { return !released_.load(std::memory_order_acquire); } + + private: + static void CUDART_CB hold(void* data) { + ParkedStream* self = static_cast<ParkedStream*>(data); + std::unique_lock<std::mutex> lock(self->mutex_); + self->cv_.wait_for(lock, self->timeout_, [self] { return self->unpark_; }); + self->released_.store(true, std::memory_order_release); + } + + cudaStream_t stream_; + std::chrono::milliseconds timeout_; + std::mutex mutex_; + std::condition_variable cv_; + bool unpark_ = false; + std::atomic<bool> released_{false}; +}; +#endif + +#ifdef CUSOLVER_VERSION +// cusolverDnXpotrf in cuSOLVER 11.4.1 (CUDA 11.8) and 11.4.2 (CUDA 12.0) reports +// success on a matrix that is not positive definite and returns a NaN factor; +// 11.4.4 (CUDA 12.1) and later set info > 0 again. Tests that expect +// NumericalIssue from such an LLT pass the reported info here: on an affected +// library this prints a warning and returns true, and the caller skips the check. +inline bool cusolver_potrf_missed_non_spd(Eigen::ComputationInfo info) { + constexpr int kFirstVersionReportingNonSpd = 11404; + int version = 0; + if (cusolverGetVersion(&version) != CUSOLVER_STATUS_SUCCESS || version >= kFirstVersionReportingNonSpd || + info != Eigen::Success) { + return false; + } + std::cout << "WARNING: cuSOLVER " << version / 1000 << '.' << version / 100 % 10 << '.' << version % 100 + << " reported success for the potrf of a matrix that is not positive definite (defect in 11.4.1 and " + "11.4.2, fixed in 11.4.4); skipping the NumericalIssue check." + << std::endl; + return true; +} #endif #ifdef CUDSS_VERSION
diff --git a/contrib/test/GPU/runtime_checks.cpp b/contrib/test/GPU/runtime_checks.cpp new file mode 100644 index 0000000..ed07672 --- /dev/null +++ b/contrib/test/GPU/runtime_checks.cpp
@@ -0,0 +1,152 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +// A failed CUDA runtime or library call has to reach EIGEN_GPU_CHECK_FAILED in a +// release build as much as in a debug one (runtime_checks_ndebug.cpp compiles +// this file with EIGEN_NO_DEBUG). The hook is overridden here to record the +// failure instead of stopping the process. + +#include <string> + +struct RecordedGpuFailure { + std::string error; + std::string expression; + std::string file; + int line = 0; +}; +static RecordedGpuFailure g_last_failure; +static int g_num_failures = 0; +static bool g_throw_failures = false; // throw the record, as a user's hook may + +static void record_gpu_failure(const char* error, const char* expression, const char* file, int line) { + ++g_num_failures; + g_last_failure.error = error; + g_last_failure.expression = expression; + g_last_failure.file = file; + g_last_failure.line = line; + if (g_throw_failures) throw g_last_failure; +} +#define EIGEN_GPU_CHECK_FAILED(error, expression, file, line) ::record_gpu_failure(error, expression, file, line) + +#define EIGEN_USE_GPU +#include "main.h" +#include <contrib/Eigen/GPU> + +#include "./gpu_test_helpers.h" + +using namespace Eigen; + +static bool starts_with(const std::string& s, const std::string& prefix) { + return s.compare(0, prefix.size(), prefix) == 0; +} + +static bool ends_with(const std::string& s, const std::string& suffix) { + return s.size() >= suffix.size() && s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +// A runtime call that fails inside the module is reported with the call's own +// text and location: downloading a view of a null pointer copies from null. +void test_runtime_call() { + gpu::Context ctx; + g_num_failures = 0; + gpu::DeviceMatrix<double> bogus = gpu::DeviceMatrix<double>::view(nullptr, 4, 1); + (void)bogus.toHost(ctx.stream()); + VERIFY_IS_EQUAL(g_num_failures, 1); + VERIFY_IS_EQUAL(g_last_failure.error, std::string("cudaErrorInvalidValue")); + VERIFY(starts_with(g_last_failure.expression, "cudaMemcpyAsync")); + VERIFY(ends_with(g_last_failure.file, "DeviceMatrix.h")); + VERIFY(g_last_failure.line > 0); + + // The context is still usable, and a successful call reports nothing. + const VectorXd x = VectorXd::Random(8); + VERIFY_IS_APPROX(VectorXd(gpu::DeviceMatrix<double>::fromHost(x, ctx.stream()).toHost(ctx.stream())), x); + VERIFY_IS_EQUAL(g_num_failures, 1); +} + +// cuBLAS before 11.6.1 cannot name a status, see cublas_check_failed(). +static std::string cublas_status_text(cublasStatus_t status, const char* name) { +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 110601 + EIGEN_UNUSED_VARIABLE(status); + return name; +#else + EIGEN_UNUSED_VARIABLE(name); + return "cuBLAS status " + std::to_string(int(status)); +#endif +} + +// Each library's check macro reports a failed status, named where the library +// can name it. The macros are given the statuses directly: what a library +// returns for invalid arguments differs between versions, and some calls +// dereference a null handle instead of returning a status. +void test_library_checks() { + g_num_failures = 0; + EIGEN_CUBLAS_CHECK(CUBLAS_STATUS_SUCCESS); + EIGEN_NPP_CHECK(NPP_NO_OPERATION_WARNING); // positive NPP statuses are warnings + VERIFY_IS_EQUAL(g_num_failures, 0); + + EIGEN_CUBLAS_CHECK(CUBLAS_STATUS_INVALID_VALUE); + VERIFY_IS_EQUAL(g_num_failures, 1); + VERIFY_IS_EQUAL(g_last_failure.error, cublas_status_text(CUBLAS_STATUS_INVALID_VALUE, "CUBLAS_STATUS_INVALID_VALUE")); + VERIFY_IS_EQUAL(g_last_failure.expression, std::string("CUBLAS_STATUS_INVALID_VALUE")); + VERIFY(ends_with(g_last_failure.file, "runtime_checks.cpp")); + VERIFY(g_last_failure.line > 0); + + EIGEN_CUBLASLT_CHECK(CUBLAS_STATUS_NOT_SUPPORTED); + VERIFY_IS_EQUAL(g_num_failures, 2); + VERIFY_IS_EQUAL(g_last_failure.error, cublas_status_text(CUBLAS_STATUS_NOT_SUPPORTED, "CUBLAS_STATUS_NOT_SUPPORTED")); + + EIGEN_CUSOLVER_CHECK(CUSOLVER_STATUS_INVALID_VALUE); + VERIFY_IS_EQUAL(g_num_failures, 3); + VERIFY_IS_EQUAL(g_last_failure.error, std::string("CUSOLVER_STATUS_INVALID_VALUE")); + + EIGEN_CUSPARSE_CHECK(CUSPARSE_STATUS_INVALID_VALUE); + VERIFY_IS_EQUAL(g_num_failures, 4); + VERIFY_IS_EQUAL(g_last_failure.error, std::string("CUSPARSE_STATUS_INVALID_VALUE")); + + EIGEN_CUFFT_CHECK(CUFFT_INVALID_PLAN); + VERIFY_IS_EQUAL(g_num_failures, 5); + VERIFY_IS_EQUAL(g_last_failure.error, "cuFFT status " + std::to_string(int(CUFFT_INVALID_PLAN))); + + EIGEN_NPP_CHECK(NPP_NULL_POINTER_ERROR); + VERIFY_IS_EQUAL(g_num_failures, 6); + VERIFY_IS_EQUAL(g_last_failure.error, "NPP status " + std::to_string(int(NPP_NULL_POINTER_ERROR))); +} + +// A hook that throws unwinds through the module with the cuBLAS handle back in +// its pointer mode, so that later host-scalar calls on the context still work. +void test_throwing_hook() { + gpu::Context ctx; + cublasPointerMode_t mode = CUBLAS_POINTER_MODE_DEVICE; + EIGEN_CUBLAS_CHECK(cublasGetPointerMode(ctx.cublasHandle(), &mode)); + VERIFY(mode == CUBLAS_POINTER_MODE_HOST); + + bool thrown = false; + g_throw_failures = true; + try { + gpu::internal::with_device_pointer_mode(ctx.cublasHandle(), [] { EIGEN_CUBLAS_CHECK(CUBLAS_STATUS_ALLOC_FAILED); }); + } catch (const RecordedGpuFailure&) { + thrown = true; + } + g_throw_failures = false; + VERIFY(thrown); + EIGEN_CUBLAS_CHECK(cublasGetPointerMode(ctx.cublasHandle(), &mode)); + VERIFY(mode == CUBLAS_POINTER_MODE_HOST); + + const VectorXd x = VectorXd::Random(16); + gpu::DeviceMatrix<double> d_x = gpu::DeviceMatrix<double>::fromHost(x, ctx.stream()); + d_x.scale(ctx, 2.0); + VERIFY_IS_APPROX(VectorXd(d_x.toHost(ctx.stream())), VectorXd(2.0 * x)); +} + +EIGEN_DECLARE_TEST(gpu_runtime_checks) { + gpu_test::require_cuda_device(); + CALL_SUBTEST(test_runtime_call()); + CALL_SUBTEST(test_library_checks()); + CALL_SUBTEST(test_throwing_hook()); +}
diff --git a/contrib/test/GPU/runtime_checks_ndebug.cpp b/contrib/test/GPU/runtime_checks_ndebug.cpp new file mode 100644 index 0000000..db5ebd0 --- /dev/null +++ b/contrib/test/GPU/runtime_checks_ndebug.cpp
@@ -0,0 +1,12 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +// The checks have to fire with assertions compiled out, as in a release build. +#define EIGEN_NO_DEBUG 1 +#include "runtime_checks.cpp" // NOLINT(bugprone-suspicious-include): the same suite under EIGEN_NO_DEBUG.
diff --git a/contrib/test/NNLS.cpp b/contrib/test/NNLS.cpp index 1101310..0c23aff 100644 --- a/contrib/test/NNLS.cpp +++ b/contrib/test/NNLS.cpp
@@ -102,6 +102,7 @@ verify_nnls_optimality(A, b, x, tolerance); } +template <typename = void> void test_nnls_handles_zero_rhs() { // // SETUP @@ -125,6 +126,7 @@ VERIFY_IS_EQUAL(x, VectorXd::Zero(cols)); } +template <typename = void> void test_nnls_handles_Mx0_matrix() { // // SETUP @@ -147,6 +149,7 @@ VERIFY_IS_EQUAL(x.size(), 0); } +template <typename = void> void test_nnls_handles_0x0_matrix() { // // SETUP @@ -168,6 +171,7 @@ VERIFY_IS_EQUAL(x.size(), 0); } +template <typename = void> void test_nnls_handles_0xN_matrix() { const MatrixXd A(0, 3); const VectorXd b(0); @@ -180,6 +184,7 @@ VERIFY_IS_EQUAL(x, VectorXd::Zero(3)); } +template <typename = void> void test_nnls_handles_dependent_columns() { // // SETUP @@ -214,6 +219,7 @@ } } +template <typename = void> void test_nnls_handles_wide_matrix() { // // SETUP @@ -248,6 +254,7 @@ } } +template <typename = void> void test_nnls_wide_matrix_at_row_capacity() { Matrix<double, 2, 3> A; A << 1, 0, -1, 0, 1, -1; @@ -262,6 +269,7 @@ verify_nnls_optimality(A, b, x, nnls.tolerance()); } +template <typename = void> void test_nnls_does_not_report_false_success_at_row_capacity() { Matrix<double, 2, 4> A; A << 9000, 9000, -3000, 1e-11, 9000, 9000, -3000, -1e-11; @@ -277,6 +285,7 @@ VERIFY_IS_EQUAL(nnls.iterations(), 2); } +template <typename = void> void test_nnls_reports_nonfinite_inactive_solution() { const Matrix<double, 3, 2> A = Matrix<double, 3, 2>::Zero(); Vector3d b; @@ -289,6 +298,7 @@ } // 4x2 problem, unconstrained solution positive +template <typename = void> void test_nnls_known_1() { Matrix<double, 4, 2> A(4, 2); Matrix<double, 4, 1> b(4); @@ -301,6 +311,7 @@ } // 4x3 problem, unconstrained solution positive +template <typename = void> void test_nnls_known_2() { Matrix<double, 4, 3> A(4, 3); Matrix<double, 4, 1> b(4); @@ -314,6 +325,7 @@ } // Simple 4x4 problem, unconstrained solution non-negative +template <typename = void> void test_nnls_known_3() { Matrix<double, 4, 4> A(4, 4); Matrix<double, 4, 1> b(4); @@ -327,6 +339,7 @@ } // Simple 4x3 problem, unconstrained solution non-negative +template <typename = void> void test_nnls_known_4() { Matrix<double, 4, 3> A(4, 3); Matrix<double, 4, 1> b(4); @@ -340,6 +353,7 @@ } // Simple 4x3 problem, unconstrained solution indefinite +template <typename = void> void test_nnls_known_5() { Matrix<double, 4, 3> A(4, 3); Matrix<double, 4, 1> b(4); @@ -353,14 +367,16 @@ test_nnls_known_solution(A, b, x); } +template <typename = void> void test_nnls_small_reference_problems() { - test_nnls_known_1(); - test_nnls_known_2(); - test_nnls_known_3(); - test_nnls_known_4(); - test_nnls_known_5(); + test_nnls_known_1<>(); + test_nnls_known_2<>(); + test_nnls_known_3<>(); + test_nnls_known_4<>(); + test_nnls_known_5<>(); } +template <typename = void> void test_nnls_with_half_precision() { // The random matrix generation tools don't work with `half`, // so here's a simpler setup mostly just to check that NNLS compiles & runs with custom scalar types. @@ -379,6 +395,7 @@ verify_nnls_optimality(A, b, x, half(64) * NumTraits<half>::epsilon()); } +template <typename = void> void test_nnls_special_case_solves_in_zero_iterations() { // The particular NNLS algorithm that is implemented starts with all variables // in the active set. @@ -406,6 +423,7 @@ VERIFY(nnls.iterations() == 0); } +template <typename = void> void test_nnls_special_case_solves_in_n_iterations() { // The particular NNLS algorithm that is implemented starts with all variables // in the active set and then adds one variable to the inactive set each iteration. @@ -430,6 +448,7 @@ VERIFY(nnls.iterations() == n); } +template <typename = void> void test_nnls_returns_NoConvergence_when_maxIterations_is_too_low() { // Using the special case that takes `n` iterations, // from `test_nnls_special_case_solves_in_n_iterations`, @@ -451,6 +470,7 @@ VERIFY(nnls.iterations() == max_iters); } +template <typename = void> void test_nnls_default_maxIterations_is_twice_column_count() { const Index cols = internal::random<Index>(1, EIGEN_TEST_MAX_SIZE); const Index rows = internal::random<Index>(cols, EIGEN_TEST_MAX_SIZE); @@ -461,6 +481,7 @@ VERIFY_IS_EQUAL(nnls.maxIterations(), 2 * cols); } +template <typename = void> void test_nnls_does_not_allocate_during_solve() { const Index cols = internal::random<Index>(1, EIGEN_TEST_MAX_SIZE); const Index rows = internal::random<Index>(cols, EIGEN_TEST_MAX_SIZE); @@ -474,6 +495,7 @@ internal::set_is_malloc_allowed(true); } +template <typename = void> void test_nnls_repeated_calls_to_compute_and_solve() { const Index cols2 = internal::random<Index>(1, EIGEN_TEST_MAX_SIZE); const Index rows2 = internal::random<Index>(cols2, EIGEN_TEST_MAX_SIZE); @@ -501,35 +523,35 @@ EIGEN_DECLARE_TEST(NNLS) { // Small matrices with known solutions: - CALL_SUBTEST_1(test_nnls_small_reference_problems()); - CALL_SUBTEST_1(test_nnls_handles_Mx0_matrix()); - CALL_SUBTEST_1(test_nnls_handles_0x0_matrix()); - CALL_SUBTEST_1(test_nnls_handles_0xN_matrix()); - CALL_SUBTEST_6(test_nnls_wide_matrix_at_row_capacity()); - CALL_SUBTEST_6(test_nnls_does_not_report_false_success_at_row_capacity()); - CALL_SUBTEST_6(test_nnls_reports_nonfinite_inactive_solution()); + CALL_SUBTEST_1(test_nnls_small_reference_problems<>()); + CALL_SUBTEST_1(test_nnls_handles_Mx0_matrix<>()); + CALL_SUBTEST_1(test_nnls_handles_0x0_matrix<>()); + CALL_SUBTEST_1(test_nnls_handles_0xN_matrix<>()); + CALL_SUBTEST_6(test_nnls_wide_matrix_at_row_capacity<>()); + CALL_SUBTEST_6(test_nnls_does_not_report_false_success_at_row_capacity<>()); + CALL_SUBTEST_6(test_nnls_reports_nonfinite_inactive_solution<>()); for (int i = 0; i < g_repeat; i++) { // Essential NNLS properties, across different types. CALL_SUBTEST_2(test_nnls_random_problem(MatrixXf())); CALL_SUBTEST_3(test_nnls_random_problem(MatrixXd())); CALL_SUBTEST_4(test_nnls_random_problem(Matrix<double, 12, 5>())); - CALL_SUBTEST_5(test_nnls_with_half_precision()); + CALL_SUBTEST_5(test_nnls_with_half_precision<>()); // Robustness tests: - CALL_SUBTEST_6(test_nnls_handles_zero_rhs()); - CALL_SUBTEST_6(test_nnls_handles_dependent_columns()); - CALL_SUBTEST_6(test_nnls_handles_wide_matrix()); + CALL_SUBTEST_6(test_nnls_handles_zero_rhs<>()); + CALL_SUBTEST_6(test_nnls_handles_dependent_columns<>()); + CALL_SUBTEST_6(test_nnls_handles_wide_matrix<>()); // Properties specific to the implementation, // not NNLS in general. - CALL_SUBTEST_7(test_nnls_special_case_solves_in_zero_iterations()); - CALL_SUBTEST_7(test_nnls_special_case_solves_in_n_iterations()); - CALL_SUBTEST_7(test_nnls_returns_NoConvergence_when_maxIterations_is_too_low()); - CALL_SUBTEST_7(test_nnls_default_maxIterations_is_twice_column_count()); - CALL_SUBTEST_8(test_nnls_repeated_calls_to_compute_and_solve()); + CALL_SUBTEST_7(test_nnls_special_case_solves_in_zero_iterations<>()); + CALL_SUBTEST_7(test_nnls_special_case_solves_in_n_iterations<>()); + CALL_SUBTEST_7(test_nnls_returns_NoConvergence_when_maxIterations_is_too_low<>()); + CALL_SUBTEST_7(test_nnls_default_maxIterations_is_twice_column_count<>()); + CALL_SUBTEST_8(test_nnls_repeated_calls_to_compute_and_solve<>()); // This test fails. It hits allocations in HouseholderSequence.h - // test_nnls_does_not_allocate_during_solve(); + // test_nnls_does_not_allocate_during_solve<>(); } }
diff --git a/contrib/test/autodiff.cpp b/contrib/test/autodiff.cpp index 02ffa2f..2f54c72 100644 --- a/contrib/test/autodiff.cpp +++ b/contrib/test/autodiff.cpp
@@ -10,7 +10,10 @@ #include "main.h" #include <Eigen/Jacobi> +#include <Eigen/QR> +#include <Eigen/SVD> #include <contrib/Eigen/AutoDiff> +#include <contrib/Eigen/Splines> template <typename Scalar> EIGEN_DONT_INLINE Scalar foo(const Scalar& x, const Scalar& y) { @@ -196,7 +199,6 @@ VERIFY_IS_APPROX(res.value(), foo(p.x(), p.y())); } -// TODO also check actual derivatives! template <int> void test_autodiff_vector() { Vector2f p = Vector2f::Random(); @@ -210,6 +212,36 @@ VERIFY_IS_APPROX(res.value(), foo(p)); } +// foo(p) = |p - (-1, 1)| + 2 |p|^2, so grad foo(p) = (p - (-1, 1)) / |p - (-1, 1)| + 4 p. +inline void check_autodiff_vector_derivative(const Vector2f& p, float expected_value, + const Vector2f& expected_gradient) { + using AD = AutoDiffScalar<Vector2f>; + using VectorAD = Matrix<AD, 2, 1>; + VectorAD ap = p.cast<AD>(); + ap.x().derivatives() = Vector2f::UnitX(); + ap.y().derivatives() = Vector2f::UnitY(); + const AD res = foo<VectorAD>(ap); + VERIFY_IS_APPROX(res.value(), expected_value); + VERIFY_IS_APPROX(res.derivatives(), expected_gradient); +} + +template <int> +void test_autodiff_vector_derivative() { + Vector2f p = Vector2f::Random(); + // Stay away from the kink of the norm term at (-1, 1). + if (numext::abs(p.x() + 1.0f) < 0.01f) p.x() = -0.9f; + const float x = p.x(), y = p.y(); + const float norm = std::sqrt((x + 1.0f) * (x + 1.0f) + (y - 1.0f) * (y - 1.0f)); + check_autodiff_vector_derivative(p, norm + 2.0f * (x * x + y * y), + Vector2f((x + 1.0f) / norm + 4.0f * x, (y - 1.0f) / norm + 4.0f * y)); +} + +template <int> +void test_autodiff_vector_derivative_specific_values() { + check_autodiff_vector_derivative(Vector2f(2.0f, 3.0f), 29.605551f, Vector2f(8.832050f, 12.554700f)); + check_autodiff_vector_derivative(Vector2f(-3.4f, 7.2f), 133.448308f, Vector2f(-13.960994f, 29.732568f)); +} + template <int> void test_autodiff_jacobian() { CALL_SUBTEST((forward_jacobian(TestFunc1<double, 2, 2>()))); @@ -373,13 +405,87 @@ } } +// Householder and Jacobi kernels must materialize AutoDiffScalar expressions before numext::sqrt and need +// isnan/isinf overloads. +template <typename Derivatives> +void test_autodiff_householder_qr() { + using AD = AutoDiffScalar<Derivatives>; + Matrix<AD, 2, 2> a; + for (int k = 0; k < 4; ++k) a(k / 2, k % 2) = AD(double(k + 1), Derivatives(Vector4d::Unit(k))); + HouseholderQR<Matrix<AD, 2, 2>> qr(a); + const AD r00 = qr.matrixQR()(0, 0); + const double n = std::sqrt(10.0); + VERIFY_IS_APPROX(numext::abs(r00.value()), n); + VERIFY_IS_APPROX(r00.derivatives(), Vector4d(1.0 / n, 0, 3.0 / n, 0) * numext::sign(r00.value())); + const Matrix<AD, 2, 2> q = qr.householderQ(); + const Matrix<AD, 2, 2> r = qr.matrixQR().template triangularView<Upper>(); + const Matrix<AD, 2, 2> qra = q * r; + for (int k = 0; k < 4; ++k) { + VERIFY_IS_APPROX(qra(k / 2, k % 2).value(), a(k / 2, k % 2).value()); + VERIFY_IS_APPROX(qra(k / 2, k % 2).derivatives(), Derivatives(Vector4d::Unit(k))); + } +} + +template <typename Derivatives> +void test_autodiff_jacobi_svd() { + using AD = AutoDiffScalar<Derivatives>; + Matrix<AD, 2, 2> a; + for (int k = 0; k < 4; ++k) a(k / 2, k % 2) = AD(double(k + 1), Derivatives(Vector4d::Unit(k))); + JacobiSVD<Matrix<AD, 2, 2>, ComputeFullU | ComputeFullV> svd(a); + const Matrix<AD, 2, 2> usv = svd.matrixU() * svd.singularValues().asDiagonal() * svd.matrixV().transpose(); + for (int k = 0; k < 4; ++k) { + VERIFY_IS_APPROX(usv(k / 2, k % 2).value(), a(k / 2, k % 2).value()); + VERIFY_IS_APPROX(usv(k / 2, k % 2).derivatives(), Derivatives(Vector4d::Unit(k))); + } +} + +// SplineFitting::Interpolate solves its collocation system with HouseholderQR. +template <typename Derivatives> +void test_autodiff_spline_interpolate() { + using AD = AutoDiffScalar<Derivatives>; + using Spline1 = Spline<AD, 1>; + const Vector4d values(1, 2, 5, 10); + Matrix<AD, 1, 4> points; + for (int k = 0; k < 4; ++k) points(k) = AD(values(k), Derivatives(Vector4d::Unit(k))); + Matrix<AD, 1, 4> parameters; + parameters << AD(0.0), AD(1.0 / 3), AD(2.0 / 3), AD(1.0); + const Spline1 spline = SplineFitting<Spline1>::Interpolate(points, 2, parameters); + for (int k = 0; k < 4; ++k) { + const AD y = spline(parameters(k))(0); + VERIFY_IS_APPROX(y.value(), values(k)); + VERIFY_IS_APPROX(y.derivatives(), Derivatives(Vector4d::Unit(k))); + } + // With fixed parameters the interpolant is linear in the points: y(u) = sum_k (dy/dp_k) p_k. + const AD y = spline(AD(0.5))(0); + VERIFY_IS_APPROX(y.value(), Vector4d(y.derivatives()).dot(values)); + VERIFY_IS_APPROX(Vector4d(y.derivatives()).sum(), 1.0); + + // Chord-length parameters depend on the points; the total derivative of y(u_k(p), p) is still e_k. + Matrix<AD, 2, 4> points2; + for (int k = 0; k < 4; ++k) { + points2(0, k) = AD(double(k), Derivatives(Vector4d::Zero())); + points2(1, k) = points(k); + } + using Spline2 = Spline<AD, 2>; + typename Spline2::KnotVectorType chordLengths; + ChordLengths(points2, chordLengths); + const Spline2 spline2 = SplineFitting<Spline2>::Interpolate(points2, 2, chordLengths); + for (int k = 0; k < 4; ++k) { + const AD y2 = spline2(chordLengths(k))(1); + VERIFY_IS_APPROX(y2.value(), values(k)); + VERIFY_IS_APPROX(y2.derivatives(), Derivatives(Vector4d::Unit(k))); + } +} + EIGEN_DECLARE_TEST(autodiff) { for (int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(test_autodiff_scalar<1>()); CALL_SUBTEST_2(test_autodiff_vector<1>()); + CALL_SUBTEST_2(test_autodiff_vector_derivative<1>()); CALL_SUBTEST_3(test_autodiff_jacobian<1>()); CALL_SUBTEST_4(test_autodiff_hessian<1>()); } + CALL_SUBTEST_2(test_autodiff_vector_derivative_specific_values<1>()); CALL_SUBTEST_5(bug_1222()); CALL_SUBTEST_5(bug_1223()); @@ -389,4 +495,10 @@ CALL_SUBTEST_5(test_autodiff_selfadjoint_l1norm()); CALL_SUBTEST_6(test_autodiff_makegivens<Vector2d>()); CALL_SUBTEST_6(test_autodiff_makegivens<VectorXd>()); + CALL_SUBTEST_7(test_autodiff_householder_qr<Vector4d>()); + CALL_SUBTEST_7(test_autodiff_householder_qr<VectorXd>()); + CALL_SUBTEST_7(test_autodiff_jacobi_svd<Vector4d>()); + CALL_SUBTEST_7(test_autodiff_jacobi_svd<VectorXd>()); + CALL_SUBTEST_7(test_autodiff_spline_interpolate<Vector4d>()); + CALL_SUBTEST_7(test_autodiff_spline_interpolate<VectorXd>()); }
diff --git a/contrib/test/fft_test_shared.h b/contrib/test/fft_test_shared.h index 12e8044..6f47725 100644 --- a/contrib/test/fft_test_shared.h +++ b/contrib/test/fft_test_shared.h
@@ -38,16 +38,21 @@ inline complex<long double> promote(double x) { return complex<long double>((long double)x); } inline complex<long double> promote(long double x) { return complex<long double>((long double)x); } +// Reference DFT X[k0] = sum_k1 x[k1] w^(k0 k1 mod n), w = exp(-2 pi i / n). Reducing the exponent in +// integers indexes n precomputed twiddles with arguments in (-2 pi, 0], so the O(n^2) sum evaluates no +// exponential; evaluating exp at the unreduced phases -2 pi k0 k1 / n dominated the whole test. template <typename VT1, typename VT2> long double fft_rmse(const VT1& fftbuf, const VT2& timebuf) { long double totalpower = 0; long double difpower = 0; long double pi = acos((long double)-1); + const size_t n = (size_t)timebuf.size(); + std::vector<complex<long double>> twiddle(n); + for (size_t m = 0; m < n; ++m) twiddle[m] = polar((long double)1, (long double)(-2.) * pi * m / n); for (size_t k0 = 0; k0 < (size_t)fftbuf.size(); ++k0) { complex<long double> acc = 0; - long double phinc = (long double)(-2.) * k0 * pi / timebuf.size(); - for (size_t k1 = 0; k1 < (size_t)timebuf.size(); ++k1) { - acc += promote(timebuf[k1]) * exp(complex<long double>(0, k1 * phinc)); + for (size_t k1 = 0; k1 < n; ++k1) { + acc += promote(timebuf[k1]) * twiddle[(k0 * k1) % n]; } totalpower += numext::abs2(acc); complex<long double> x = promote(fftbuf[k0]);
diff --git a/contrib/test/polynomialsolver.cpp b/contrib/test/polynomialsolver.cpp index 0708b8c..a9f8306 100644 --- a/contrib/test/polynomialsolver.cpp +++ b/contrib/test/polynomialsolver.cpp
@@ -172,6 +172,7 @@ verify_polynomialsolver_sugar(psolve, numext::sqrt(test_precision<RealScalar>())); } +template <typename = void> void polynomialsolver_sugar_cluster() { Matrix<float, 7, 1> roots; roots << -0.8f, 0.2f, 0.2001f, 0.5f, 0.7f, 0.9f, 1.0f; @@ -180,6 +181,7 @@ evalSolverSugarFunction<7>(poly, roots); } +template <typename = void> void polynomialsolver_sugar_filtering() { Vector4d poly; poly << 0, 1, 0, 1; @@ -304,6 +306,7 @@ // At |z| = 3 rounding dominates the residual of the pair 0.1 +- 3i, which is still an accurate root; comparing that // residual with the one at the real part 0.1 once reported the pair as a double real root. +template <typename = void> void polynomialsolver_complex_pair_kept() { Matrix<std::complex<float>, 12, 1> roots; roots << std::complex<float>(0.1f, 3.0f), std::complex<float>(0.1f, -3.0f), 0.2f, -0.2f, 0.4f, -0.4f, 0.6f, -0.6f, @@ -322,6 +325,7 @@ // The companion eigenvalue of a root at 1e-12 carries an absolute error of order eps, a backward error of about // 3e3 eps; refinement restores the root's full relative accuracy. +template <typename = void> void polynomialsolver_tiny_root() { Matrix<double, 5, 1> roots; roots << 1e-12, 0.25, -0.5, 0.75, -1.0; @@ -407,8 +411,8 @@ } EIGEN_DECLARE_TEST(polynomialsolver) { - CALL_SUBTEST_7(polynomialsolver_sugar_cluster()); - CALL_SUBTEST_13(polynomialsolver_sugar_filtering()); + CALL_SUBTEST_7(polynomialsolver_sugar_cluster<>()); + CALL_SUBTEST_13(polynomialsolver_sugar_filtering<>()); CALL_SUBTEST_18(polynomialsolver_real_starts_for_complex_pair<6>()); CALL_SUBTEST_18(polynomialsolver_real_starts_for_complex_pair<Dynamic>()); for (int i = 0; i < g_repeat; i++) { @@ -434,8 +438,8 @@ internal::random<int>(2, 13)))); CALL_SUBTEST_22((polynomialsolver_refinement_accuracy<double, long double, Dynamic>(internal::random<int>(2, 20)))); } - CALL_SUBTEST_15(polynomialsolver_complex_pair_kept()); - CALL_SUBTEST_15(polynomialsolver_tiny_root()); + CALL_SUBTEST_15(polynomialsolver_complex_pair_kept<>()); + CALL_SUBTEST_15(polynomialsolver_tiny_root<>()); CALL_SUBTEST_16(polynomialsolver_real_part_is_another_root<float>()); CALL_SUBTEST_23(polynomialsolver_real_part_is_another_root<double>()); CALL_SUBTEST_24(polynomialsolver_real_part_is_another_root<std::complex<double>>());
diff --git a/contrib/test/special_functions.cpp b/contrib/test/special_functions.cpp index 1b2efb2..5d4d03c 100644 --- a/contrib/test/special_functions.cpp +++ b/contrib/test/special_functions.cpp
@@ -162,11 +162,13 @@ CALL_SUBTEST(res = x.ndtri(); verify_component_wise(res, ref);); CALL_SUBTEST(res = ndtri(x); verify_component_wise(res, ref);); - // ndtri(normal_cdf(x)) ~= x + // ndtri(normal_cdf(x)) ~= x. A rounding error of eps * p in p = normal_cdf(x) moves ndtri(p) by eps * p / pdf(x), + // at most 3.5 eps on [-1, 1], so the error is bounded absolutely rather than relative to x, which may be near 0. + // The rest of the 16 eps covers the rounding of erf and ndtri themselves. CALL_SUBTEST(ArrayType m1 = ArrayType::Random(32); using std::sqrt; ArrayType cdf_val = (m1 / Scalar(sqrt(2.))).erf(); cdf_val = (cdf_val + Scalar(1)) / Scalar(2); - verify_component_wise(cdf_val.ndtri(), m1);); + VERIFY(max_abs_coeff(cdf_val.ndtri() - m1) <= RealScalar(16) * NumTraits<RealScalar>::epsilon());); } // Check the zeta function against scipy.special.zeta @@ -293,31 +295,32 @@ ArrayType b = (m2 * Scalar(4)).exp(); ArrayType x = m3.abs(); - // betainc(a, 1, x) == x**a + // betainc(a, 1, x) == x**a. One ulp of a subnormal x**a exceeds the relative tolerance, so the smallest normal is + // added to the scale as an absolute floor. + const Scalar tiny = (std::numeric_limits<Scalar>::min)(); CALL_SUBTEST(ArrayType test = betainc(a, one, x); ArrayType expected = x.pow(a); - verify_component_wise(test, expected);); + for (Index i = 0; i < test.size(); ++i) + VERIFY_IS_APPROX_SCALED(test.segment(i, 1), expected.segment(i, 1), expected(i) + tiny);); - // betainc(1, b, x) == 1 - (1 - x)**b - CALL_SUBTEST(ArrayType test = betainc(one, b, x); ArrayType expected = one - (one - x).pow(b); + // betainc(1, b, x) == 1 - (1 - x)**b, evaluated as -expm1(b * log1p(-x)): for small b * x the direct form cancels + // to a relative error near eps / (b * x). + CALL_SUBTEST(ArrayType test = betainc(one, b, x); ArrayType expected = -(b * (-x).log1p()).expm1(); verify_component_wise(test, expected);); // betainc(a, b, x) == 1 - betainc(b, a, 1-x) CALL_SUBTEST(ArrayType test = betainc(a, b, x) + betainc(b, a, one - x); ArrayType expected = one; verify_component_wise(test, expected);); - // betainc(a+1, b, x) = betainc(a, b, x) - x**a * (1 - x)**b / (a * beta(a, b)) - CALL_SUBTEST( - ArrayType num = x.pow(a) * (one - x).pow(b); - ArrayType denom = a * (a.lgamma() + b.lgamma() - (a + b).lgamma()).exp(); - // Add eps to rhs and lhs so that component-wise test doesn't result in - // nans when both outputs are zeros. - ArrayType expected = betainc(a, b, x) - num / denom + eps; - ArrayType test = betainc(a + one, b, x) + eps; if (sizeof(Scalar) >= 8) { // double - verify_component_wise(test, expected); - } else { - // Reason for limited test: http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1232 - verify_component_wise(test.head(8), expected.head(8)); - }); + // betainc(a+1, b, x) = betainc(a, b, x) - x**a * (1 - x)**b / (a * beta(a, b)). The difference cancels, so the + // error is bounded relative to the larger of the two terms rather than to the result, plus the smallest normal as + // the floor for subnormal results. The subtracted term is evaluated in log space: in float x**a * (1 - x)**b + // underflows while the term itself, divided by a small beta(a, b), does not. + CALL_SUBTEST(ArrayType log_beta = a.lgamma() + b.lgamma() - (a + b).lgamma(); + ArrayType term = (a * x.log() + b * (-x).log1p() - a.log() - log_beta).exp(); + ArrayType lhs = betainc(a, b, x); ArrayType scale = (lhs.max)(term) + tiny; + ArrayType expected = lhs - term; ArrayType test = betainc(a + one, b, x); + for (Index i = 0; i < test.size(); ++i) + VERIFY_IS_APPROX_SCALED(test.segment(i, 1), expected.segment(i, 1), scale(i));); // betainc(a, b+1, x) = betainc(a, b, x) + x**a * (1 - x)**b / (b * beta(a, b)) CALL_SUBTEST(
diff --git a/contrib/test/stencil.cpp b/contrib/test/stencil.cpp new file mode 100644 index 0000000..b9669ed --- /dev/null +++ b/contrib/test/stencil.cpp
@@ -0,0 +1,248 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include <complex> + +#include "main.h" +#include "fp_control.h" +#include <contrib/Eigen/NumericalDiff> + +using Eigen::Array; +using Eigen::ArrayBase; +using Eigen::Index; +using Eigen::makeStencil; +using Eigen::Stencil; + +// The verify_* helpers take ArrayBase rather than Array<Scalar, Size, 1>: MSVC 14.29 cannot +// deduce Size through Array's defaulted Options argument, which depends on it. + +// Independent oracle: for order-Derivative weights at x0 = 0 over points p_0..p_{Size-1}, +// sum_i weights[i] * p_i^k == (k == Derivative ? Derivative! : 0) for every k = 0..Size-1. This +// holds for any polynomial reproduced exactly by an (Size-1)-degree interpolant, independently of +// how Stencil's recursion is implemented. +template <int Derivative, typename Derived> +void verify_polynomial_exactness(const ArrayBase<Derived>& points) { + using Scalar = typename Derived::Scalar; + using RealScalar = typename Eigen::NumTraits<Scalar>::Real; + constexpr int Size = Derived::SizeAtCompileTime; + + const Stencil<Derivative, Scalar, Size> s(points); + const Array<Scalar, Size, 1> w = s.weights(); + const Array<Scalar, Size, 1> p = s.points(); + + for (Index i = 0; i < Size; ++i) { + VERIFY_IS_EQUAL(p[i], points[i]); + VERIFY((numext::isfinite)(w[i])); + } + + Scalar fact(1); + for (Index i = 2; i <= Derivative; ++i) fact *= Scalar(i); + + Scalar term[Size]; + for (Index i = 0; i < Size; ++i) term[i] = Scalar(1); + + for (Index k = 0; k < Size; ++k) { + Scalar acc(0); + RealScalar magSum(0); + for (Index i = 0; i < Size; ++i) { + const Scalar contribution = w[i] * term[i]; + acc += contribution; + magSum += numext::abs(contribution); + } + const Scalar expected = (k == Derivative) ? fact : Scalar(0); + const RealScalar tol = + RealScalar(50) * RealScalar(Size) * (magSum + RealScalar(1)) * Eigen::NumTraits<RealScalar>::epsilon(); + VERIFY((numext::isfinite)(acc)); + VERIFY((numext::isfinite)(tol)); + VERIFY(numext::abs(acc - expected) <= tol); + for (Index i = 0; i < Size; ++i) term[i] *= p[i]; + } +} + +void test_textbook_central() { + const Array<double, 3, 1> points{{-1.0, 0.0, 1.0}}; + const double tol = 10.0 * Eigen::NumTraits<double>::epsilon(); + + const Stencil<1, double, 3> d1(points); + const Array<double, 3, 1> w1 = d1.weights(); + VERIFY(numext::abs(w1[0] - (-0.5)) <= tol); + VERIFY(numext::abs(w1[1] - 0.0) <= tol); + VERIFY(numext::abs(w1[2] - 0.5) <= tol); + + const Stencil<2, double, 3> d2(points); + const Array<double, 3, 1> w2 = d2.weights(); + VERIFY(numext::abs(w2[0] - 1.0) <= tol); + VERIFY(numext::abs(w2[1] - (-2.0)) <= tol); + VERIFY(numext::abs(w2[2] - 1.0) <= tol); +} + +void test_textbook_forward() { + const Array<double, 3, 1> points{{0.0, 1.0, 2.0}}; + const double tol = 10.0 * Eigen::NumTraits<double>::epsilon(); + + const Stencil<1, double, 3> d1 = makeStencil<1>(points); + const Array<double, 3, 1> w1 = d1.weights(); + VERIFY(numext::abs(w1[0] - (-1.5)) <= tol); + VERIFY(numext::abs(w1[1] - 2.0) <= tol); + VERIFY(numext::abs(w1[2] - (-0.5)) <= tol); +} + +// Derivative == 0 (interpolation), and the boundary case Derivative == Size - 1, fall out of the +// same sweep since verify_polynomial_exactness checks every k = 0..Size-1 already. +void test_polynomial_exactness_uniform() { + const Array<double, 5, 1> points{{-2.0, -1.0, 0.0, 1.0, 2.0}}; + verify_polynomial_exactness<0>(points); + verify_polynomial_exactness<1>(points); + verify_polynomial_exactness<2>(points); + verify_polynomial_exactness<3>(points); + verify_polynomial_exactness<4>(points); +} + +// Widely varying spacing: exercises the "arbitrary spacing" case without relying on any internal +// reordering to keep the recursion well conditioned. +void test_polynomial_exactness_nonuniform() { + const Array<double, 5, 1> points{{-1000.0, -1.0, 0.0, 1.0, 1000.0}}; + verify_polynomial_exactness<0>(points); + verify_polynomial_exactness<1>(points); + verify_polynomial_exactness<2>(points); + verify_polynomial_exactness<4>(points); +} + +// Points that do not include the evaluation point 0. +void test_polynomial_exactness_no_zero() { + const Array<double, 4, 1> points{{-3.0, -1.0, 2.0, 5.0}}; + verify_polynomial_exactness<0>(points); + verify_polynomial_exactness<1>(points); + verify_polynomial_exactness<3>(points); +} + +void test_polynomial_exactness_float() { + const Array<float, 4, 1> points{{-2.0f, -1.0f, 1.0f, 2.0f}}; + verify_polynomial_exactness<0>(points); + verify_polynomial_exactness<1>(points); + verify_polynomial_exactness<3>(points); +} + +template <int Derivative, typename Derived> +void verify_power_of_two_scaling(const ArrayBase<Derived>& points) { + using Scalar = typename Derived::Scalar; + constexpr int Size = Derived::SizeAtCompileTime; + const Stencil<Derivative, Scalar, Size> reference(points); + // Power-of-two scaling is exact while the recurrence stays in the normal range. + for (const int exponent : {-27, -20, -40, 0, 20, 27, 40}) { + const Scalar step = std::ldexp(Scalar(1), exponent); + const Array<Scalar, Size, 1> scaledPoints = points * step; + const Stencil<Derivative, Scalar, Size> scaled(scaledPoints); + for (Index i = 0; i < Size; ++i) { + const Scalar expected = std::ldexp(reference.weight(i), -Derivative * exponent); + VERIFY((numext::isfinite)(expected)); + VERIFY((numext::isfinite)(scaled.weight(i))); + VERIFY_IS_EQUAL(scaled.weight(i), expected); + } + } +} + +template <typename Scalar> +void test_scaled_points() { + const Array<Scalar, 9, 1> points{{-4, -3, -2, -1, 0, 1, 2, 3, 4}}; + const Array<Scalar, 7, 1> nonuniform{{3, -1, 0, 4, -4, 1, -2}}; + verify_power_of_two_scaling<0>(points); + verify_power_of_two_scaling<1>(points); + verify_power_of_two_scaling<2>(points); + verify_power_of_two_scaling<0>(nonuniform); + verify_power_of_two_scaling<1>(nonuniform); + verify_power_of_two_scaling<2>(nonuniform); +} + +void test_scaled_points_flush_modes() { + Eigen::forEachFlushToZeroMode([](Eigen::FlushToZeroMode) { test_scaled_points<float>(); }); +} + +void test_polynomial_exactness_complex() { + using Scalar = std::complex<double>; + const Array<Scalar, 4, 1> points{{Scalar(-1.0, 1.0), Scalar(0.0, 0.0), Scalar(1.0, 1.0), Scalar(2.0, -1.0)}}; + verify_polynomial_exactness<0>(points); + verify_polynomial_exactness<1>(points); + verify_polynomial_exactness<3>(points); +} + +// No internal reordering: weights()[i] must keep tracking points()[i] as given, so permuting the +// input permutes the output identically. +void test_order_preservation() { + const Array<double, 5, 1> points{{-1000.0, -1.0, 0.0, 1.0, 1000.0}}; + const Array<double, 5, 1> reversed{{1000.0, 1.0, 0.0, -1.0, -1000.0}}; + + const Stencil<2, double, 5> s(points); + const Stencil<2, double, 5> sReversed(reversed); + const Array<double, 5, 1> w = s.weights(); + const Array<double, 5, 1> wReversed = sReversed.weights(); + + for (Index i = 0; i < 5; ++i) { + const double tol = 100.0 * Eigen::NumTraits<double>::epsilon() * (numext::abs(w[i]) + 1.0); + VERIFY(numext::abs(w[i] - wReversed[4 - i]) <= tol); + } +} + +// Repeated points make a `points[n] - points[nu]` denominator in the recursion vanish. +void test_repeated_points_assert() { + const Array<double, 3, 1> points{{0.0, 0.0, 1.0}}; + VERIFY_RAISES_ASSERT((Stencil<1, double, 3>(points))); +} + +// The C-array constructor and weight()/point() bypass Eigen::Array and Map respectively, so unlike +// the DenseBase constructor and weights()/points(), these are real constant expressions on every +// supported C++14 compiler. +constexpr double centralPoints[] = {-1.0, 0.0, 1.0}; +constexpr Stencil<1, double, 3> centralDiff1 = makeStencil<1>(centralPoints); +static_assert(centralDiff1.weight(0) == -0.5, ""); +static_assert(centralDiff1.weight(1) == 0.0, ""); +static_assert(centralDiff1.weight(2) == 0.5, ""); +static_assert(centralDiff1.point(0) == -1.0, ""); +static_assert(centralDiff1.point(1) == 0.0, ""); +static_assert(centralDiff1.point(2) == 1.0, ""); + +constexpr float smallStep = 1.0f / 134217728.0f; // 2^-27 +constexpr float unitPoints[] = {-4, -3, -2, -1, 0, 1, 2, 3, 4}; +constexpr float smallPoints[] = {-4 * smallStep, -3 * smallStep, -2 * smallStep, -smallStep, 0, + smallStep, 2 * smallStep, 3 * smallStep, 4 * smallStep}; +constexpr Stencil<2, float, 9> unitDiff2(unitPoints); +constexpr Stencil<2, float, 9> smallDiff2(smallPoints); +static_assert(smallDiff2.weight(6) == unitDiff2.weight(6) / (smallStep * smallStep), ""); +static_assert(smallDiff2.weight(7) == unitDiff2.weight(7) / (smallStep * smallStep), ""); +static_assert(smallDiff2.weight(8) == unitDiff2.weight(8) / (smallStep * smallStep), ""); + +void test_compile_time_smoke() { + const Array<double, 3, 1> w = centralDiff1.weights(); + VERIFY(numext::abs(w[0] - (-0.5)) <= Eigen::NumTraits<double>::epsilon()); + VERIFY(numext::abs(w[1] - 0.0) <= Eigen::NumTraits<double>::epsilon()); + VERIFY(numext::abs(w[2] - 0.5) <= Eigen::NumTraits<double>::epsilon()); +} + +void test_constant_member_linkage() { + using StencilType = Stencil<1, double, 3>; + // Volatile pointers retain the odr-use even in optimized test builds. + const int* volatile pointCount = &StencilType::PointCount; + const int* volatile derivativeOrder = &StencilType::DerivativeOrder; + VERIFY_IS_EQUAL(*pointCount, 3); + VERIFY_IS_EQUAL(*derivativeOrder, 1); +} + +EIGEN_DECLARE_TEST(stencil) { + CALL_SUBTEST(test_textbook_central()); + CALL_SUBTEST(test_textbook_forward()); + CALL_SUBTEST(test_polynomial_exactness_uniform()); + CALL_SUBTEST(test_polynomial_exactness_nonuniform()); + CALL_SUBTEST(test_polynomial_exactness_no_zero()); + CALL_SUBTEST(test_polynomial_exactness_float()); + CALL_SUBTEST(test_scaled_points<float>()); + CALL_SUBTEST(test_scaled_points<double>()); + CALL_SUBTEST(test_scaled_points_flush_modes()); + CALL_SUBTEST(test_polynomial_exactness_complex()); + CALL_SUBTEST(test_order_preservation()); + CALL_SUBTEST(test_repeated_points_assert()); + CALL_SUBTEST(test_compile_time_smoke()); + CALL_SUBTEST(test_constant_member_linkage()); +}
diff --git a/contrib/test/structured_bccb.cpp b/contrib/test/structured_bccb.cpp index e732513..db62a3f 100644 --- a/contrib/test/structured_bccb.cpp +++ b/contrib/test/structured_bccb.cpp
@@ -8,7 +8,7 @@ // SPDX-License-Identifier: MPL-2.0 #include "main.h" -#include "fp_control.h" +#include "structured_test_helpers.h" #include <contrib/Eigen/StructuredMatrices> @@ -278,6 +278,7 @@ // or solve expression is built. (Incompatible *fixed* sizes are rejected at // compile time by a static assertion in operator* / solve, which a runtime test // cannot exercise.) +template <typename = void> void test_bccb_dimension_asserts() { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -350,6 +351,7 @@ VERIFY(C.solve(b).isZero()); } +template <typename = void> void test_bccb_nan_propagation(Index n2, Index n1) { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -363,136 +365,8 @@ VERIFY(!(x.array() == x.array()).all()); } -// Inputs at the very top (and bottom) of the exponent range: the transforms sum -// up to N terms, so without scaling the FFT intermediates overflow even though -// every entry of the true result is representable. The apply path rescales the -// symbol and each right-hand-side column by exact powers of two and folds the -// exponent back at the end. -void test_bccb_finite_overflow() { - typedef Matrix<double, Dynamic, 1> Vec; - typedef Matrix<double, Dynamic, Dynamic> Mat; - - const double huge = (std::numeric_limits<double>::max)() / 16; - // The 2-D FFT round trip costs a few ulps per entry; a norm-based comparison - // would itself overflow at these magnitudes, hence the entrywise bound. - const double kFftRoundTripTol = 100 * NumTraits<double>::epsilon(); - - // Reviewer repro: a 36x36 identity BCCB applied to a vector of DBL_MAX/16 - // must return the vector, not 36 NaNs. - Mat G = Mat::Zero(6, 6); - G(0, 0) = 1.0; - Bccb<double> C(G); - Vec x = Vec::Constant(36, huge); - Vec y = C * x; - VERIFY(y.allFinite()); - VERIFY(((y - x).array().abs() <= kFftRoundTripTol * x.array().abs()).all()); - - // A huge generator makes a huge symbol: the scaling must come from the symbol - // side as well. - Mat Gh = Mat::Zero(6, 6); - Gh(0, 0) = huge; // C == huge * Identity - Bccb<double> Ch(Gh); - Vec ones = Vec::Ones(36); - Vec z = Ch * ones; - VERIFY(z.allFinite()); - VERIFY(((z.array() - huge).abs() <= kFftRoundTripTol * huge).all()); - - // The inverse symbol of a tiny operator is huge; solve() shares the scaled - // apply path. - Mat Gt = Mat::Zero(6, 6); - Gt(0, 0) = 1.0 / huge; - Bccb<double> Ct(Gt); - Vec w = Ct.solve(ones); - VERIFY(w.allFinite()); - VERIFY(((w.array() - huge).abs() <= kFftRoundTripTol * huge).all()); - - // An exactly zero column maps to an exactly zero column (short-circuited - // before any scaling), while nonzero columns are still transformed. - Mat B(36, 2); - B.col(0).setConstant(huge); - B.col(1).setZero(); - Mat Y = C * B; - VERIFY(Y.col(1).isZero()); - VERIFY(((Y.col(0) - x).array().abs() <= kFftRoundTripTol * x.array().abs()).all()); - - // A genuine NaN among huge inputs keeps propagating; such right-hand sides - // take the direct kernel (see test_bccb_nonfinite_product for the entrywise - // semantics). - Vec xn = x; - xn[7] = std::numeric_limits<double>::quiet_NaN(); - Vec yn = C * xn; - VERIFY(!(yn.array() == yn.array()).all()); -} - -// The scaling exponents are derived from component-wise magnitudes: a finite -// complex value near the overflow threshold has a non-representable modulus, -// which would otherwise disable the scaling and turn an exactly representable -// product into NaN. -template <typename RealScalar> -void test_bccb_fft_complex_boundary(Index n2, Index n1) { - typedef std::complex<RealScalar> Complex; - typedef Matrix<RealScalar, Dynamic, Dynamic> RMat; - typedef Matrix<Complex, Dynamic, 1> CVec; - typedef Matrix<Complex, Dynamic, Dynamic> CMat; - const RealScalar kFftRoundTripTol = RealScalar(100) * NumTraits<RealScalar>::epsilon(); - const RealScalar big = RealScalar(0.75) * (std::numeric_limits<RealScalar>::max)(); - const Index N = n1 * n2; - - // Identity BCCB with a complex generator: the product returns the right-hand - // side unchanged even though |x_k| overflows. - CMat G = CMat::Zero(n2, n1); - G(0, 0) = Complex(1); - Bccb<Complex> C(G); - CVec x = CVec::Constant(N, Complex(big, big)); - CVec y = C * x; - VERIFY(y.allFinite()); - VERIFY(((y - x).cwiseAbs() / big).maxCoeff() <= kFftRoundTripTol); - - // A real identity operator applied to the same complex right-hand side takes - // the mixed-scalar product path. - RMat Gr = RMat::Zero(n2, n1); - Gr(0, 0) = RealScalar(1); - Bccb<RealScalar> Cr(Gr); - y = Cr * x; - VERIFY(y.allFinite()); - VERIFY(((y - x).cwiseAbs() / big).maxCoeff() <= kFftRoundTripTol); -} - -// Entrywise IEEE comparison for the non-finite tests: NaNs match NaNs, -// infinities match by value (sign included), finite entries match to roundoff. -// VERIFY_IS_APPROX would reject any output containing NaN. -template <typename D1, typename D2> -bool ieee_entrywise_match(const D1& a, const D2& b) { - if (a.rows() != b.rows() || a.cols() != b.cols()) return false; - for (Index j = 0; j < a.cols(); ++j) - for (Index i = 0; i < a.rows(); ++i) { - const typename D1::Scalar x = a(i, j), y = b(i, j); - if (x == y) continue; // finite match or same-signed infinities - if ((numext::isnan)(x) && (numext::isnan)(y)) continue; // both NaN - if (!test_isApprox(x, y)) return false; // finite roundoff - } - return true; -} - -// Scalar-loop product: the mathematically transparent IEEE reference for the -// non-finite tests. Eigen's own vectorized complex kernels can smear a single -// infinity into NaN (Inf - Inf across the split real/imaginary accumulators), so -// the dense product is not a faithful entrywise reference for non-finite data. -template <typename Scalar> -Matrix<Scalar, Dynamic, 1> reference_product_ieee(const Matrix<Scalar, Dynamic, Dynamic>& A, - const Matrix<Scalar, Dynamic, 1>& x) { - Matrix<Scalar, Dynamic, 1> y(A.rows()); - for (Index i = 0; i < A.rows(); ++i) { - Scalar acc(0); - for (Index j = 0; j < A.cols(); ++j) acc += A(i, j) * x[j]; - y[i] = acc; - } - return y; -} - -// A single Inf or NaN in the data must propagate like the reference product -- -// through the dot products that touch it -- instead of being smeared into NaNs -// across the whole output by the transforms. +// An Inf or NaN in the data is never lost and does not leak into other columns +// (see nonfinite_covers_reference()). template <typename Scalar> void test_bccb_nonfinite_product(Index n2, Index n1) { typedef typename NumTraits<Scalar>::Real RealScalar; @@ -506,59 +380,46 @@ Bccb<Scalar> C(G); Mat dense = reference_bccb<Scalar>(G); - // Inf in the right-hand side. Vec x = Vec::Random(N); x[N / 2] = Scalar(inf); - VERIFY(ieee_entrywise_match((C * x).eval(), reference_product_ieee(dense, x))); - - // NaN in the right-hand side. + VERIFY(nonfinite_covers_reference((C * x).eval(), reference_product_ieee(dense, x))); Vec xn = Vec::Random(N); xn[N - 1] = Scalar(nan); - VERIFY(ieee_entrywise_match((C * xn).eval(), reference_product_ieee(dense, xn))); + VERIFY(nonfinite_covers_reference((C * xn).eval(), reference_product_ieee(dense, xn))); + Vec xz = Vec::Zero(N); + xz[0] = Scalar(nan); + VERIFY(nonfinite_covers_reference((C * xz).eval(), reference_product_ieee(dense, xz))); - // Mixed multi-column right-hand side: the non-finite column falls back to the - // direct kernel individually while the finite column keeps the FFT path. Mat Xm(N, 2); Xm.col(0) = Vec::Random(N); Xm.col(1) = x; Mat Ym = C * Xm; VERIFY_IS_APPROX(Ym.col(0).eval(), (dense * Xm.col(0)).eval()); - VERIFY(ieee_entrywise_match(Ym.col(1).eval(), reference_product_ieee(dense, Vec(Xm.col(1))))); + VERIFY(nonfinite_covers_reference(Ym.col(1).eval(), reference_product_ieee(dense, Vec(Xm.col(1))))); - // A zero column carrying a single NaN must not take the zero-column shortcut: - // the fast-max routing scan can miss a NaN among zeros (an Inf always - // surfaces), so the shortcut rechecks exactly and such a column falls back. - Vec xz = Vec::Zero(N); - xz[0] = Scalar(nan); - VERIFY(ieee_entrywise_match((C * xz).eval(), reference_product_ieee(dense, xz))); - - // Inf in the generating array: the operator itself is non-finite, whatever the - // right-hand side. Mat G2 = Mat::Random(n2, n1); G2(n2 / 2, n1 / 2) = Scalar(-inf); Bccb<Scalar> C2(G2); Mat dense2 = reference_bccb<Scalar>(G2); Vec x2 = Vec::Random(N); - VERIFY(ieee_entrywise_match((C2 * x2).eval(), reference_product_ieee(dense2, x2))); + VERIFY(nonfinite_covers_reference((C2 * x2).eval(), reference_product_ieee(dense2, x2))); - // Non-finite right-hand sides of solve() apply the pseudo-inverse -- itself a - // BCCB operator -- through the direct kernel, so the Inf propagates entrywise - // instead of NaN-ing the whole output through the transforms. The operator is - // diagonally dominant, so no symbol entry is thresholded and the pseudo-inverse - // coincides with inverse(), whose dense form serves as the reference matrix. Mat Gd = Mat::Random(n2, n1); Gd(0, 0) += Scalar(RealScalar(2 * N)); Bccb<Scalar> Cd(Gd); - Mat pinv = Mat(Cd.inverse()); - Vec binf = Vec::Random(N); - binf[N / 3] = Scalar(inf); - VERIFY(ieee_entrywise_match(Cd.solve(binf), reference_product_ieee(pinv, binf))); + Mat B(N, 2); + B.col(0) = Vec::Random(N); + B.col(1) = x; + Mat X = Cd.solve(B); + VERIFY_IS_APPROX((reference_bccb<Scalar>(Gd) * X.col(0)).eval(), Vec(B.col(0))); + VERIFY(!X.col(1).allFinite()); } -// An all-zero right-hand side must not short-circuit to an exact zero when the -// operator holds non-finite data: every row of a BCCB touches every generator +// An all-zero right-hand side does not produce an exact zero when the operator +// holds non-finite data: every row of a BCCB touches every generator // entry, so each result entry is a dot product with an Inf coefficient times // zero -- NaN under IEEE, not zero. +template <typename = void> void test_bccb_nonfinite_zero_rhs(Index n2, Index n1) { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -572,7 +433,7 @@ const Vec z = Vec::Zero(N); Vec y = C * z; VERIFY(y.hasNaN()); - VERIFY(ieee_entrywise_match(y, reference_product_ieee(dense, z))); + VERIFY(nonfinite_covers_reference(y, reference_product_ieee(dense, z))); // The same holds for solve(): the symbol of a non-finite operator holds NaN, // so the pseudo-inverse applied to a zero right-hand side is NaN, not zero. @@ -584,6 +445,7 @@ // plain product of the eigenvalues in FFT order overflows to infinity (or // underflows to an exact zero) partway through. Pins the balanced accumulation // in determinant(). +template <typename = void> void test_bccb_determinant_scaled() { typedef std::complex<double> Complex; typedef Matrix<Complex, Dynamic, Dynamic> CMat; @@ -637,6 +499,7 @@ // smallest-normal symbol entry is still inverted -- the comparison is strict, // matching SVDBase::rank(), which likewise reports rank one -- a subnormal entry // is treated as an exact zero, and non-finite entries count as non-zero. +template <typename = void> void test_bccb_rank_boundaries() { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -662,66 +525,6 @@ } } -// A finite complex symbol entry near the overflow threshold has a -// non-representable modulus. The rank threshold used to be computed from the raw -// moduli, turning it into infinity: the rank was under-reported and solve() -// zeroed valid Fourier modes. Both are now evaluated in an exactly rescaled -// frame. -void test_bccb_rank_complex_boundary() { - typedef std::complex<double> Complex; - typedef Matrix<Complex, Dynamic, 1> CVec; - typedef Matrix<Complex, Dynamic, Dynamic> CMat; - const double mx = (std::numeric_limits<double>::max)(); - - // n2 = 2, n1 = 1: the symbol is exactly [g0 + g1, g0 - g1], so pick the - // generator from the desired spectrum. |s0| overflows while both of its - // components are finite. - const Complex s0(0.75 * mx, 0.75 * mx), s1(1e300, 0.0); - CMat G(2, 1); - G(0, 0) = (s0 + s1) * 0.5; - G(1, 0) = (s0 - s1) * 0.5; - Bccb<Complex> C(G); - VERIFY_IS_EQUAL(C.rank(), 2); - - // The second Fourier mode must be inverted, not zeroed: a product of a small - // vector solves back to that vector (the accuracy is limited by the condition - // number |s0| / |s1| ~ 2.5e8). - CVec x0(2); - x0[0] = Complex(1e-10, -2e-10); - x0[1] = Complex(-3e-10, 1e-10); - CVec b = C * x0; - VERIFY(b.allFinite()); - CVec x = C.solve(b); - VERIFY(((x - x0).cwiseAbs().maxCoeff() / x0.cwiseAbs().maxCoeff()) <= 1e-6); - - // A genuinely negligible second entry still truncates in the scaled frame. - CMat G2(2, 1); - G2(0, 0) = (s0 + Complex(1)) * 0.5; - G2(1, 0) = (s0 - Complex(1)) * 0.5; - VERIFY_IS_EQUAL(Bccb<Complex>(G2).rank(), 1); -} - -// A single 2^-e frame factor is itself subnormal once the frame exceeds the -// exponent range, and reads as zero under flush-to-zero: every scaled modulus -// and the threshold collapse together, the rank is over-reported, and solve() -// inverts a mode it should have truncated. Hence the two exact factors. -void test_bccb_rank_flush_to_zero() { - ScopedFlushToZero flush_to_zero; - if (!flush_to_zero.isSupported()) return; - - using Complex = std::complex<double>; - using CMat = Matrix<Complex, Dynamic, Dynamic>; - const double mx = (std::numeric_limits<double>::max)(); - - // Spectrum [s0, 1] with |s0| at the overflow boundary: the second mode is - // negligible against it, so the operator is rank one. - const Complex s0(0.75 * mx, 0.75 * mx); - CMat G(2, 1); - G(0, 0) = (s0 + Complex(1)) * 0.5; - G(1, 0) = (s0 - Complex(1)) * 0.5; - VERIFY_IS_EQUAL(Bccb<Complex>(G).rank(), 1); -} - template <typename Scalar> void test_bccb_eigen(Index n2, Index n1) { typedef typename NumTraits<Scalar>::Real RealScalar; @@ -841,143 +644,11 @@ } template <typename RealScalar> -void test_structured_symbol_reciprocals() { - using Complex = std::complex<RealScalar>; - using CVector = Matrix<Complex, Dynamic, 1>; - using RVector = Matrix<RealScalar, Dynamic, 1>; - constexpr Index kPacketSize = internal::packet_traits<Complex>::size; - const Index count = 3 * kPacketSize + 1; - const RealScalar epsilon = NumTraits<RealScalar>::epsilon(); - const RealScalar lower = numext::ldexp(RealScalar(1), std::numeric_limits<RealScalar>::min_exponent / 2); - const RealScalar upper = numext::ldexp(RealScalar(1), std::numeric_limits<RealScalar>::max_exponent / 2 - 1); - const RealScalar minimum = (std::numeric_limits<RealScalar>::min)(); - const RealScalar maximum = (std::numeric_limits<RealScalar>::max)(); - const RealScalar infinity = NumTraits<RealScalar>::infinity(); - const RealScalar nan = NumTraits<RealScalar>::quiet_NaN(); - const RealScalar values[] = {RealScalar(0), - -RealScalar(0), - RealScalar(1), - RealScalar(-1), - lower / 2, - lower, - upper, - upper * 2, - minimum, - minimum / 2, - std::numeric_limits<RealScalar>::denorm_min(), - RealScalar(0.75) * maximum, - infinity, - nan}; - CVector symbol(count + 2), inverse(count + 2), unthresholded(count + 2); - RVector moduli(count + 2); - // Slide each special value through every lane, with unaligned inputs and a scalar tail. - for (Index offset = 0; offset <= 1; ++offset) { - for (Index lane = 0; lane < kPacketSize; ++lane) { - for (RealScalar real : values) { - for (RealScalar imag : values) { - symbol.setConstant(Complex(3, 4)); - moduli.setOnes(); - symbol(offset + lane) = Complex(real, imag); - symbol(offset + count - 1) = Complex(real, imag); - inverse.setConstant(Complex(9)); - internal::structured_symbol_reciprocal<RealScalar>(symbol.data() + offset, moduli.data() + offset, - RealScalar(1), inverse.data() + offset, count); - unthresholded = symbol; - internal::structured_symbol_reciprocal(unthresholded.data() + offset, count); - VERIFY_IS_EQUAL(unthresholded(offset + count), symbol(offset + count)); - if (offset) VERIFY_IS_EQUAL(unthresholded(0), symbol(0)); - VERIFY_IS_EQUAL(inverse(offset + count), Complex(9)); - if (offset) VERIFY_IS_EQUAL(inverse(0), Complex(9)); - for (Index k = offset; k < offset + count; ++k) { - const Complex expected = internal::structured_scaled_reciprocal(symbol(k)); - for (const CVector* result : {&inverse, &unthresholded}) { - for (Index component = 0; component < 2; ++component) { - const RealScalar actual = component ? (*result)(k).imag() : (*result)(k).real(); - const RealScalar reference = component ? expected.imag() : expected.real(); - if ((numext::isnan)(reference)) { - VERIFY((numext::isnan)(actual)); - } else if ((numext::isinf)(reference) || reference == RealScalar(0)) { - VERIFY_IS_EQUAL(actual, reference); - VERIFY_IS_EQUAL(std::signbit(actual), std::signbit(reference)); - } else { - VERIFY((numext::isfinite)(actual)); - VERIFY(numext::abs(actual / reference - RealScalar(1)) <= RealScalar(8) * epsilon); - } - } - } - } - } - } - } - } - - // Positive denominator: two squares, their sum and a division, plus reference rounding. - for (Index trial = 0; trial < 100; ++trial) { - for (Index k = 0; k < count; ++k) { - const int exponent = internal::random<int>(std::numeric_limits<RealScalar>::min_exponent / 2, - std::numeric_limits<RealScalar>::max_exponent / 2 - 1); - const RealScalar re = numext::ldexp(internal::random<RealScalar>(RealScalar(0.5), RealScalar(1)), exponent); - const RealScalar im = numext::ldexp(internal::random<RealScalar>(RealScalar(-1), RealScalar(-0.5)), exponent); - symbol(k) = Complex(re, im); - } - moduli.setOnes(); - internal::structured_symbol_reciprocal<RealScalar>(symbol.data(), moduli.data(), RealScalar(1), inverse.data(), - count); - for (Index k = 0; k < count; ++k) { - const long double re = static_cast<long double>(symbol(k).real()); - const long double im = static_cast<long double>(symbol(k).imag()); - const long double scale = (std::max)(std::abs(re), std::abs(im)); - const long double a = re / scale, b = im / scale; - const long double denom = a * a + b * b; - const long double expected[] = {(a / denom) / scale, (-b / denom) / scale}; - for (Index component = 0; component < 2; ++component) { - const long double actual = static_cast<long double>(component ? inverse(k).imag() : inverse(k).real()); - VERIFY(std::abs(actual / expected[component] - 1) <= 8 * static_cast<long double>(epsilon)); - } - } - } - - symbol.setConstant(Complex(3, 4)); - moduli.setOnes(); - for (Index k = 0; k < count; k += 2) { - symbol(k) = Complex(0); - moduli(k) = RealScalar(0); - } - std::fenv_t saved_environment; - const bool environment_saved = std::feholdexcept(&saved_environment) == 0; - internal::structured_symbol_reciprocal<RealScalar>(symbol.data(), moduli.data(), RealScalar(1), inverse.data(), - count); - if (environment_saved) { - const int exceptions = std::fetestexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW); - std::fesetenv(&saved_environment); - VERIFY_IS_EQUAL(exceptions, 0); - } - for (Index k = 0; k < count; ++k) { - if (k % 2 == 0) { - VERIFY_IS_EQUAL(inverse(k), Complex(0)); - VERIFY(!std::signbit(inverse(k).real())); - VERIFY(!std::signbit(inverse(k).imag())); - } else { - const Complex expected(RealScalar(3) / RealScalar(25), -RealScalar(4) / RealScalar(25)); - VERIFY(numext::abs(inverse(k) - expected) <= RealScalar(8) * epsilon); - } - } - // NaN moduli, like exact equality above, must remain in the inverted set. - symbol.setConstant(Complex(3, 4)); - moduli.setConstant(nan); - internal::structured_symbol_reciprocal<RealScalar>(symbol.data(), moduli.data(), RealScalar(1), inverse.data(), - count); - VERIFY((inverse.head(count).array().abs() > RealScalar(0)).all()); - internal::structured_symbol_reciprocal<RealScalar>(nullptr, nullptr, RealScalar(1), nullptr, 0); - internal::structured_symbol_reciprocal<RealScalar>(nullptr, 0); -} - -template <typename RealScalar> -void test_structured_packet_reciprocals() { +void test_constant_symbol_solves() { using Complex = std::complex<RealScalar>; using CMatrix = Matrix<Complex, Dynamic, Dynamic>; CMatrix generator = CMatrix::Zero(3, 5); - generator(0, 0) = Complex(3, 4); // Constant non-real symbol exercises packets and the tail. + generator(0, 0) = Complex(3, 4); // A constant non-real symbol. const Bccb<Complex> op(generator); const CMatrix expected = CMatrix::Random(15, 2); const CMatrix rhs = generator(0, 0) * expected; @@ -1008,33 +679,9 @@ VERIFY_IS_EQUAL(circulant.rank(), 1); const Complex circulant_mode = circulant.inverse().symbol()(1); VERIFY(numext::abs(circulant_mode * epsilon - Complex(1)) <= RealScalar(8) * epsilon); - - // 1/(a+ia) = (0.5/a)(1-i), even when an unscaled denominator overflows. - const RealScalar a = RealScalar(0.75) * (std::numeric_limits<RealScalar>::max)(); - for (Index n : {Index(1), Index(48)}) { - column.setZero(n); - column(0) = Complex(a, a); - const CVector circulant_inverse = Circulant<Complex>(column).inverse().symbol(); - VERIFY(circulant_inverse.allFinite()); - volatile RealScalar minimum = (std::numeric_limits<RealScalar>::min)(); - if (!numext::is_exactly_zero(minimum * RealScalar(0.5))) { - for (Index k = 0; k < n; ++k) { - VERIFY(numext::abs(circulant_inverse(k).real() * a - RealScalar(0.5)) <= RealScalar(64) * epsilon); - VERIFY(numext::abs(circulant_inverse(k).imag() * a + RealScalar(0.5)) <= RealScalar(64) * epsilon); - } - } - } } EIGEN_DECLARE_TEST(structured_bccb) { - STATIC_CHECK((!internal::structured_reciprocal_packet_traits<long double>::Vectorizable)); -#ifdef EIGEN_VECTORIZE_SSE2 - STATIC_CHECK((internal::structured_reciprocal_packet_traits<float>::Vectorizable)); - STATIC_CHECK((internal::structured_reciprocal_packet_traits<double>::Vectorizable)); -#else - STATIC_CHECK((!internal::structured_reciprocal_packet_traits<float>::Vectorizable)); - STATIC_CHECK((!internal::structured_reciprocal_packet_traits<double>::Vectorizable)); -#endif for (int i = 0; i < g_repeat; ++i) { // Products, dense assignment, coefficient access: scalar tier (N <= 32) and // 2-D FFT tier, including single-row/column-of-blocks degenerate shapes. @@ -1065,7 +712,7 @@ CALL_SUBTEST_3((test_bccb_rank_deficient<std::complex<double>>(6, 8, 4))); CALL_SUBTEST_3((test_bccb_rank_deficient<std::complex<float>>(4, 5, 2))); CALL_SUBTEST_3((test_bccb_zero<double>(4, 5))); - CALL_SUBTEST_3(test_bccb_nan_propagation(6, 7)); + CALL_SUBTEST_3(test_bccb_nan_propagation<>(6, 7)); // Closed-form eigendecomposition, SVD, inverse, determinant. CALL_SUBTEST_4((test_bccb_eigen<double>(4, 5))); @@ -1105,31 +752,21 @@ CALL_SUBTEST_6((test_bccb_mixed_scalar<double>(3, 4))); CALL_SUBTEST_6((test_bccb_mixed_scalar<double>(6, 8))); CALL_SUBTEST_6((test_bccb_mixed_scalar<float>(8, 8))); - CALL_SUBTEST_6(test_bccb_dimension_asserts()); + CALL_SUBTEST_6(test_bccb_dimension_asserts<>()); - // Finite-range robustness: scaled transforms, the complex overflow boundary, - // balanced determinant accumulation, and the rank threshold boundary. - CALL_SUBTEST_7(test_bccb_finite_overflow()); - CALL_SUBTEST_7((test_bccb_fft_complex_boundary<double>(6, 8))); - CALL_SUBTEST_7((test_bccb_fft_complex_boundary<float>(6, 8))); - CALL_SUBTEST_7(test_bccb_determinant_scaled()); - CALL_SUBTEST_7(test_bccb_rank_boundaries()); - CALL_SUBTEST_7(test_bccb_rank_complex_boundary()); - CALL_SUBTEST_7(test_bccb_rank_flush_to_zero()); + // Balanced determinant accumulation and the rank threshold boundary. + CALL_SUBTEST_7(test_bccb_determinant_scaled<>()); + CALL_SUBTEST_7(test_bccb_rank_boundaries<>()); - // Entrywise Inf/NaN propagation: FFT-sized operators must fall back to the - // direct kernel; small ones are IEEE-exact already. A zero right-hand side + // Inf/NaN propagation on the direct and FFT tiers. A zero right-hand side // under a non-finite operator yields NaN, not zero. CALL_SUBTEST_7((test_bccb_nonfinite_product<double>(6, 8))); CALL_SUBTEST_7((test_bccb_nonfinite_product<double>(3, 4))); CALL_SUBTEST_7((test_bccb_nonfinite_product<std::complex<double>>(6, 8))); - CALL_SUBTEST_7(test_bccb_nonfinite_zero_rhs(6, 8)); + CALL_SUBTEST_7(test_bccb_nonfinite_zero_rhs<>(6, 8)); - CALL_SUBTEST_8(test_structured_symbol_reciprocals<float>()); - CALL_SUBTEST_8(test_structured_symbol_reciprocals<double>()); - CALL_SUBTEST_8(test_structured_symbol_reciprocals<long double>()); - CALL_SUBTEST_8(test_structured_packet_reciprocals<float>()); - CALL_SUBTEST_8(test_structured_packet_reciprocals<double>()); + CALL_SUBTEST_8(test_constant_symbol_solves<float>()); + CALL_SUBTEST_8(test_constant_symbol_solves<double>()); CALL_SUBTEST_8(test_circulant_inverse_modes<float>()); CALL_SUBTEST_8(test_circulant_inverse_modes<double>()); }
diff --git a/contrib/test/structured_cauchy.cpp b/contrib/test/structured_cauchy.cpp index 68f7685..31fb833 100644 --- a/contrib/test/structured_cauchy.cpp +++ b/contrib/test/structured_cauchy.cpp
@@ -23,18 +23,6 @@ return dense; } -// Reference dense Cauchy built entry-wise through the guarded reciprocal, the -// single helper every coefficient evaluation must agree with at the overflow -// boundary. -template <typename Scalar> -Matrix<Scalar, Dynamic, Dynamic> reference_cauchy_guarded(const Matrix<Scalar, Dynamic, 1>& x, - const Matrix<Scalar, Dynamic, 1>& y) { - Matrix<Scalar, Dynamic, Dynamic> dense(x.size(), y.size()); - for (Index j = 0; j < y.size(); ++j) - for (Index i = 0; i < x.size(); ++i) dense(i, j) = internal::cauchy_reciprocal_diff(x[i], y[j]); - return dense; -} - // Separated node sets: x in [2,3], y in [0,1], so all denominators are in [1,3]. template <typename Scalar> void separated_nodes(Index m, Index n, Matrix<Scalar, Dynamic, 1>& x, Matrix<Scalar, Dynamic, 1>& y) { @@ -79,6 +67,30 @@ Vec w0 = w; w.noalias() += C * v; VERIFY_IS_APPROX(w, (w0 + dense * v).eval()); + + // Several right-hand sides share each reciprocal column; -= passes alpha = -1. + Mat W = Mat::Random(m, 3); + const Mat W0 = W; + W.noalias() -= C * V; + VERIFY_IS_APPROX(W, (W0 - dense * V).eval()); +} + +// With 100 right-hand sides, 1100 rows exceed the 1024-row block floor of the +// multi-column product, so the rows split into a full and a ragged block. +template <typename Scalar> +void test_cauchy_product_row_blocks() { + typedef Matrix<Scalar, Dynamic, 1> Vec; + typedef Matrix<Scalar, Dynamic, Dynamic> Mat; + const Index m = 1100, n = 8, r = 100; + Vec x, y; + separated_nodes<Scalar>(m, n, x, y); + Cauchy<Scalar> C(x, y); + const Mat dense = reference_cauchy<Scalar>(x, y); + const Mat V = Mat::Random(n, r); + Mat W = Mat::Random(m, r); + const Mat W0 = W; + W.noalias() -= C * V; + VERIFY_IS_APPROX(W, (W0 - dense * V).eval()); } template <typename Scalar> @@ -143,6 +155,7 @@ // The Hilbert matrix as a Cauchy matrix (x_i = i+1, y_j = -j): deterministic // residual bounds far below what the astronomical conditioning would allow a // forward-error test, plus the closed-form determinant against the dense LU one. +template <typename = void> void test_cauchy_hilbert() { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -179,6 +192,7 @@ // Clustered row nodes make leading minors nearly singular: partial pivoting must // keep the factorization backward stable (residual check). +template <typename = void> void test_cauchy_lu_pivoting(Index n) { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -200,6 +214,7 @@ // A duplicated row node makes two rows identical, hence the matrix exactly // singular; a zero pivot must survive partial pivoting and be reported. +template <typename = void> void test_cauchy_lu_singular() { typedef Matrix<double, Dynamic, 1> Vec; Vec x(5), y(5); @@ -210,44 +225,22 @@ VERIFY(lu.info() == NumericalIssue); } -// Generator mantissas can overflow even when every matrix entry and LU factor -// is finite. Exponent-tracked generators must preserve the cancellation against -// the node reciprocal, and still expose an exact duplicate-row singularity. +// Exactly singular and non-finite pivots are reported through info(). template <typename Scalar> -void test_cauchy_lu_scaled_generators() { +void test_cauchy_lu_issues() { typedef typename NumTraits<Scalar>::Real RealScalar; typedef Matrix<Scalar, Dynamic, 1> Vec; - typedef Matrix<Scalar, Dynamic, Dynamic> Mat; - Vec x(2), y(2), b(2); - const RealScalar P = std::ldexp(RealScalar(1), 1000); - const RealScalar t = std::ldexp(RealScalar(1), -1000); - x << Scalar(0), Scalar(-P); - y << Scalar(P), Scalar(-t); - b << Scalar(1), Scalar(0.25); - Cauchy<Scalar> C(x, y); - Mat dense = C; - VERIFY(dense.allFinite()); - CauchyLU<Scalar> lu(C); - VERIFY(lu.info() == Success); - Vec u = lu.solve(b); - VERIFY(u.allFinite()); - const RealScalar tol = RealScalar(64) * NumTraits<RealScalar>::epsilon(); - VERIFY((dense * u - b).cwiseAbs().maxCoeff() <= tol * b.cwiseAbs().maxCoeff()); - - Vec xs(2); + Vec xs(2), y(2); xs << Scalar(0), Scalar(0); - Cauchy<Scalar> Cs(xs, y); - Mat denseSingular = Cs; - VERIFY(denseSingular.allFinite()); - CauchyLU<Scalar> singularLu(Cs); + y << Scalar(1), Scalar(-0.5); + CauchyLU<Scalar> singularLu(Cauchy<Scalar>(xs, y)); VERIFY(singularLu.info() == NumericalIssue); Vec xi(1), yi(1); xi << Scalar(0); yi << Scalar(-std::numeric_limits<RealScalar>::denorm_min()); - Cauchy<Scalar> Ci(xi, yi); - CauchyLU<Scalar> nonfiniteLu(Ci); + CauchyLU<Scalar> nonfiniteLu(Cauchy<Scalar>(xi, yi)); VERIFY(nonfiniteLu.info() == NumericalIssue); } @@ -416,6 +409,12 @@ w.noalias() += C * v; VERIFY_IS_APPROX(w, (w0 + dense * v).eval()); + CMat V = CMat::Random(n, 3); + CMat W = CMat::Random(m, 3); + const CMat W0 = W; + W.noalias() -= C * V; + VERIFY_IS_APPROX(W, (W0 - dense * V).eval()); + CVec xc, yc; separated_nodes<Complex>(m, n, xc, yc); Cauchy<Complex> Cc(xc, yc); @@ -428,6 +427,7 @@ // Wide-dynamic-range determinants: the closed-form factors overflow or underflow // individually while the determinant itself is representable, so the balanced // m * 2^e accumulation must carry the exponent past the intermediate extremes. +template <typename = void> void test_cauchy_determinant_range() { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -498,100 +498,21 @@ } } -// Node differences at the overflow boundary: forming x_j - x_i or x_i - y_j can -// overflow to Inf even though every matrix entry is finite and unexceptional. -// Such factors must enter the balanced accumulation through the exact -// halved-operand recomputation, so only the determinant's own overflow or -// underflow is visible in the result -- saturated to a zero or infinity of the -// mathematically correct sign. -void test_cauchy_determinant_overflow_boundary() { +// Genuine overflow: clustered tiny nodes push the determinant past the +// representable range while every factor stays finite; the accumulated +// exponent must saturate to a correctly signed infinity, in both signs. +template <typename = void> +void test_cauchy_determinant_saturation() { typedef Matrix<double, Dynamic, 1> Vec; - const double M = 0.6 * (std::numeric_limits<double>::max)(); - - // Reviewer reproducer: every coefficient 1/(x_i - y_j) is finite and the exact - // determinant (2M)(-0.8M) / (0.7056 M^4) underflows, but the numerator - // difference x_1 - x_0 = 1.2 * DBL_MAX overflows if formed naively (the old - // code returned -Inf). The result must be a zero of the correct sign: one - // negative numerator factor against two negative denominator factors. + const double t = std::ldexp(1.0, -537); { Vec x(2), y(2); - x << -M, M; - y << -0.4 * M, 0.4 * M; - const double det = Cauchy<double>(x, y).determinant(); - VERIFY(det == 0.0 && std::signbit(det)); - } - - // Sign flip of the same configuration: swapping the y nodes negates the - // determinant, so the underflow must land on +0. - { - Vec x(2), y(2); - x << -M, M; - y << 0.4 * M, -0.4 * M; - const double det = Cauchy<double>(x, y).determinant(); - VERIFY(det == 0.0 && !std::signbit(det)); - } - - // Purely imaginary nodes of the same magnitudes: the halved-operand - // recomputation applies componentwise to complex nodes. Scaling every node by - // i multiplies the 2x2 determinant by i^2 / i^4 = -1, so this underflow lands - // on a real part of +0 (the value is real: the four denominator divisions - // rotate the accumulation back onto the real axis). - { - typedef std::complex<double> Cplx; - Matrix<Cplx, Dynamic, 1> x(2), y(2); - x << Cplx(0.0, -M), Cplx(0.0, M); - y << Cplx(0.0, -0.4 * M), Cplx(0.0, 0.4 * M); - const Cplx det = Cauchy<Cplx>(x, y).determinant(); - VERIFY(numext::real(det) == 0.0 && numext::imag(det) == 0.0); - VERIFY(!std::signbit(numext::real(det))); - } - - // A representable determinant whose evaluation crosses the boundary, guard on - // a denominator factor (accumulated exponent decremented): with P = 2^1023, - // x = [-P, 0], y = [P, c], the difference x_0 - y_0 = -2^1024 overflows, yet - // det = (P - c) / (2 P c (P + c)), which for c = 2^-60 is 2^-964 up to a - // relative correction c/P ~ 2^-1083, far below roundoff. Every rounded factor - // is a power of two, so the balanced accumulation is exact here. - { - const double P = std::ldexp(1.0, 1023); - const double c = std::ldexp(1.0, -60); - Vec x(2), y(2); - x << -P, 0.0; - y << P, c; - const double det = Cauchy<double>(x, y).determinant(); - VERIFY_IS_APPROX(det, std::ldexp(1.0, -964)); - } - - // Guard on a numerator factor (accumulated exponent incremented): - // x = [-P, P, 0], y = [c, -c, d] with c = 2^1000 and d = 2^-1050. The - // numerator difference x_1 - x_0 = 2^1024 overflows; the determinant - // -4 P^3 (c^2 - d^2) / ((P^2 - c^2)^2 (P^2 - d^2) c d) equals - // -2^-1017 / (1 - 2^-46)^2 up to relative corrections of order 2^-2050. - { - const double P = std::ldexp(1.0, 1023); - const double c = std::ldexp(1.0, 1000); - const double d = std::ldexp(1.0, -1050); - Vec x(3), y(3); - x << -P, P, 0.0; - y << c, -c, d; - const double det = Cauchy<double>(x, y).determinant(); - const double r = 1.0 - std::ldexp(1.0, -46); // 1 - (c/P)^2 - VERIFY_IS_APPROX(det, -std::ldexp(1.0, -1017) / (r * r)); - } - - // Genuine overflow: clustered tiny nodes push the determinant past the - // representable range while every factor stays finite; the accumulated - // exponent must saturate to a correctly signed infinity, in both signs. - { - const double t = std::ldexp(1.0, -537); - Vec x(2), y(2); x << 0.0, 3.0 * t; y << t, 2.0 * t; // det = -3 / (4 t^2) ~ -1.6e323 const double det = Cauchy<double>(x, y).determinant(); VERIFY((numext::isinf)(det) && std::signbit(det)); } { - const double t = std::ldexp(1.0, -537); Vec x(2), y(2); x << 0.0, 3.0 * t; y << 2.0 * t, t; // swapped y nodes: det = +3 / (4 t^2) @@ -600,127 +521,6 @@ } } -// Boundary nodes in every coefficient evaluation: when x_i - y_j overflows for -// finite nodes, a naively formed coefficient collapses to 1/Inf = 0 while the -// true value is a representable subnormal. coeff(), the dense materialization, -// the products and the CauchyLU factorization must all produce the guarded -// value -- and agree with determinant(), which derives the same quantity -// through its balanced accumulation. -void test_cauchy_boundary_coefficients() { - typedef Matrix<double, Dynamic, 1> Vec; - typedef Matrix<double, Dynamic, Dynamic> Mat; - const double X = (std::numeric_limits<double>::max)(); - const double e = std::ldexp(1.0, -1025); // 1/(2*DBL_MAX), correctly rounded (subnormal) - - // 1x1 reviewer case: the single coefficient is 1/(2*DBL_MAX) = 2^-1025 - // (2.7813423231340017e-309). Every step is correctly rounded IEEE arithmetic - // with a power-of-two result, so the checks are exact equalities. - { - Vec x(1), y(1); - x << X; - y << -X; - Cauchy<double> C(x, y); - VERIFY_IS_EQUAL(C.coeff(0, 0), e); - Mat d = C; - VERIFY_IS_EQUAL(d(0, 0), e); - Mat acc = Mat::Zero(1, 1); - acc += C; // addTo - VERIFY_IS_EQUAL(acc(0, 0), e); - acc -= C; // subTo - VERIFY_IS_EQUAL(acc(0, 0), 0.0); - Vec p = C * Vec::Ones(1); - VERIFY_IS_EQUAL(p[0], e); - // determinant() reaches 2^-1025 through the balanced accumulation; the - // guarded coefficient matches it exactly. - VERIFY_IS_EQUAL(C.determinant(), e); - // The old code materialized a zero coefficient and reported the (regular) - // matrix as singular; the guarded pivot is subnormal but non-zero. - CauchyLU<double> lu(C); - VERIFY(lu.info() == Success); - Vec u = lu.solve(p); // p = C * [1]; recovered exactly (e / e = 1) - VERIFY_IS_EQUAL(u[0], 1.0); - } - - // A non-power-of-two boundary difference catches double rounding: computing - // (1/t)*0.5 is one subnormal ULP above the correctly rounded 0.5/t. MPFR gives - // the expected value below for x=1.1777865466541931e308 and - // y=-1.7435684109415042e308, represented by their exact double bit patterns. - { - Vec x(1), y(1); - x << numext::bit_cast<double>(numext::uint64_t(0x7fe4f71daafe5a86ull)); - y << numext::bit_cast<double>(numext::uint64_t(0xffef095b3493b755ull)); - const double expected = numext::bit_cast<double>(numext::uint64_t(0x00027621a9baa547ull)); - Cauchy<double> C(x, y); - VERIFY_IS_EQUAL(C.coeff(0, 0), expected); - Mat dense = C; - VERIFY_IS_EQUAL(dense(0, 0), expected); - Vec product = C * Vec::Ones(1); - VERIFY_IS_EQUAL(product[0], expected); - VERIFY_IS_EQUAL(C.determinant(), expected); - CauchyLU<double> lu(C); - VERIFY(lu.info() == Success); - Vec rhs(1); - rhs << expected; - Vec solution = lu.solve(rhs); - VERIFY_IS_EQUAL(solution[0], 1.0); - } - - // 2x2 with boundary pairs among moderate ones: three of the four differences - // overflow (2X and two 1.5X), one stays finite (X). All APIs agree with the - // reference built from the guarded reciprocal; scaled by its magnitude the - // matrix is well conditioned (cond ~ 38), so the GKO solve -- whose column - // generation and generator updates cross the boundary too -- recovers the - // solution accurately. - { - Vec x(2), y(2); - x << X, X / 2; - y << -X, -X / 2; - Cauchy<double> C(x, y); - Mat ref = reference_cauchy_guarded<double>(x, y); - VERIFY_IS_EQUAL(ref(0, 0), e); // 1/(2*DBL_MAX) - VERIFY_IS_EQUAL(ref(1, 1), std::ldexp(1.0, -1024)); // 1/DBL_MAX - VERIFY(ref.allFinite()); - VERIFY((ref.array() != 0.0).all()); - for (Index j = 0; j < 2; ++j) - for (Index i = 0; i < 2; ++i) VERIFY_IS_EQUAL(C.coeff(i, j), ref(i, j)); - Mat d = C; - for (Index j = 0; j < 2; ++j) - for (Index i = 0; i < 2; ++i) VERIFY_IS_EQUAL(d(i, j), ref(i, j)); - Vec v(2); - v << 0.75, -0.5; - VERIFY_IS_APPROX((C * v).eval(), (ref * v).eval()); - CauchyLU<double> lu(C); - VERIFY(lu.info() == Success); - Vec b = C * Vec::Ones(2); - Vec u = lu.solve(b); - VERIFY_IS_APPROX(u, Vec::Ones(2).eval()); - } - - // Complex nodes: purely imaginary boundary nodes overflow in the imaginary - // component of the difference; the guard applies componentwise. The paths - // sharing the helper agree exactly; the value is -i/(2*DBL_MAX) and the - // determinant derives it independently, both to within a few subnormal - // spacings (complex division rounds per component). - { - typedef std::complex<double> Cplx; - const double tiny = 4.0 * std::numeric_limits<double>::denorm_min(); - Matrix<Cplx, Dynamic, 1> xc(1), yc(1); - xc << Cplx(0.0, X); - yc << Cplx(0.0, -X); - Cauchy<Cplx> C(xc, yc); - const Cplx cval = C.coeff(0, 0); - Matrix<Cplx, Dynamic, Dynamic> dc = C; - VERIFY_IS_EQUAL(dc(0, 0), cval); - Matrix<Cplx, Dynamic, 1> pc = C * Matrix<Cplx, Dynamic, 1>::Ones(1); - VERIFY_IS_EQUAL(pc[0], cval); - VERIFY(numext::abs(numext::real(cval)) <= tiny); - VERIFY(numext::abs(numext::imag(cval) + e) <= tiny); - const Cplx det = C.determinant(); - VERIFY(numext::abs(numext::real(det) - numext::real(cval)) <= tiny); - VERIFY(numext::abs(numext::imag(det) - numext::imag(cval)) <= tiny); - } -} - template <typename Scalar, int M, int N> void test_cauchy_fixed() { typedef Matrix<Scalar, M, 1> XVec; @@ -764,6 +564,12 @@ Matrix<Scalar, M, 1> w2; w2.noalias() = C * v; VERIFY_IS_APPROX(w2, (dense * v).eval()); + + // Several right-hand sides take the fixed-size reciprocal workspace. + const Matrix<Scalar, N, 3> V = Matrix<Scalar, N, 3>::Random(); + Matrix<Scalar, M, 3> W; + W.noalias() = C * V; + VERIFY_IS_APPROX(W, (dense * V).eval()); } EIGEN_DECLARE_TEST(structured_cauchy) { @@ -776,6 +582,8 @@ CALL_SUBTEST_1((test_cauchy_product<float>(10, 10))); CALL_SUBTEST_1((test_cauchy_product<std::complex<double>>(9, 7))); CALL_SUBTEST_1((test_cauchy_product<std::complex<float>>(7, 9))); + CALL_SUBTEST_1(test_cauchy_product_row_blocks<double>()); + CALL_SUBTEST_1(test_cauchy_product_row_blocks<std::complex<double>>()); CALL_SUBTEST_1((test_cauchy_transpose<double>(10, 14))); CALL_SUBTEST_1((test_cauchy_transpose<std::complex<double>>(8, 6))); @@ -786,11 +594,11 @@ CALL_SUBTEST_2((test_cauchy_lu<double>(30))); CALL_SUBTEST_2((test_cauchy_lu<std::complex<double>>(16))); CALL_SUBTEST_2((test_cauchy_lu<float>(10))); - CALL_SUBTEST_2(test_cauchy_hilbert()); - CALL_SUBTEST_2(test_cauchy_lu_pivoting(20)); - CALL_SUBTEST_2(test_cauchy_lu_singular()); - CALL_SUBTEST_2((test_cauchy_lu_scaled_generators<double>())); - CALL_SUBTEST_2((test_cauchy_lu_scaled_generators<std::complex<double>>())); + CALL_SUBTEST_2(test_cauchy_hilbert<>()); + CALL_SUBTEST_2(test_cauchy_lu_pivoting<>(20)); + CALL_SUBTEST_2(test_cauchy_lu_singular<>()); + CALL_SUBTEST_2((test_cauchy_lu_issues<double>())); + CALL_SUBTEST_2((test_cauchy_lu_issues<std::complex<double>>())); // Closed-form determinant, symmetric generalized Hilbert, fixed sizes. CALL_SUBTEST_3((test_cauchy_determinant<double>(4))); @@ -809,8 +617,7 @@ CALL_SUBTEST_4((test_cauchy_delayed_product<std::complex<double>>(7, 10))); CALL_SUBTEST_4((test_cauchy_mixed_scalar<double>(10, 13))); CALL_SUBTEST_4((test_cauchy_mixed_scalar<float>(9, 6))); - CALL_SUBTEST_4(test_cauchy_determinant_range()); - CALL_SUBTEST_4(test_cauchy_determinant_overflow_boundary()); - CALL_SUBTEST_4(test_cauchy_boundary_coefficients()); + CALL_SUBTEST_4(test_cauchy_determinant_range<>()); + CALL_SUBTEST_4(test_cauchy_determinant_saturation<>()); } }
diff --git a/contrib/test/structured_dplr.cpp b/contrib/test/structured_dplr.cpp index b17348d..b744938 100644 --- a/contrib/test/structured_dplr.cpp +++ b/contrib/test/structured_dplr.cpp
@@ -285,6 +285,7 @@ // when the determinant itself is representable, whatever the ordering of large // and small diagonal entries. Genuinely non-representable determinants must // still saturate to infinity / zero. +template <typename = void> void test_dplr_determinant_scaled() { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -346,6 +347,7 @@ // the determinant is ~1, since D^{-1} scales the capacitance entries by the reciprocal diagonal. Both // det(D) and det(capacitance) must therefore accumulate in the balanced mantissa * 2^e form, the // latter from the LU pivots rather than the plain pivot product .determinant() would form. +template <typename = void> void test_dplr_determinant_capacitance() { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -418,118 +420,10 @@ } } -// The capacitance triple product V^H D^{-1} U can overflow in its plain association while the -// capacitance itself is representable: D^{-1} pairs the reciprocal of a tiny diagonal with a huge -// factor before the other, tiny factor pulls the product back down. The Woodbury kernels form these -// products from exactly rescaled factors so no spurious Inf/NaN reaches determinant(), solve() or -// inverse(). -void test_dplr_capacitance_overflow() { - typedef Matrix<double, Dynamic, 1> Vec; - typedef std::complex<double> Complex; - typedef Matrix<Complex, Dynamic, 1> CVec; - const double eps = NumTraits<double>::epsilon(); - - { - // The reviewer's case: D + UV^H = 1e-200 + 1e-200 * 1e200 ~ 1 and the - // capacitance 1 + 1e200 is representable, but V^H D^{-1} = 1e400 is not; - // determinant() used to return Inf and solve() NaN. - Vec d(1), u(1), v(1); - d << 1e-200; - u << 1e-200; - v << 1e200; - DiagonalPlusLowRank<double> A(d, u, v); - VERIFY((numext::isfinite)(A.capacitance()(0, 0))); - VERIFY_IS_APPROX(A.capacitance()(0, 0), 1e200); - VERIFY(numext::abs(A.determinant() - 1.0) <= 8.0 * eps); - // The Woodbury splitting routes this solve through D^{-1} b ~ 1e200 and - // cancels it back down to x ~ 1, so the amplified digits are lost to any - // association (see the solve() documentation); the fix is that the solve - // stays finite instead of turning into NaN through an infinite capacitance. - Vec b(1); - b << 1.0; - VERIFY(A.solve(b).allFinite()); - - // D^{-1} b is 1e400 here, though the operator is essentially the identity and the solution 1e200 - // is representable; without the power-of-two rescaling the first term is Inf and the correction - // turns it into NaN, which is what this pins. It cannot pin finiteness: A ~ 1 makes the correction - // cancel D^{-1} b over ~200 digits, and whether that leaves 0 or an ulp the undone scaling lifts - // back to Inf is a codegen detail (x86-64 gives the first, aarch64 the second). - Vec big_b(1); - big_b << 1e200; - VERIFY(!A.solve(big_b).array().isNaN().any()); - } - { - // The same overflow pattern in exact powers of two, where the Woodbury - // arithmetic is exact end to end: d = 2^-522, U = 2^-1025, V = 2^503 - // represent A = [2^-521] exactly; V^H D^{-1} = 2^1025 still overflows, and - // the capacitance is exactly 2. Determinant and solve must match the dense - // results for the represented matrix exactly. - Vec d(1), u(1), v(1); - d << std::ldexp(1.0, -522); - u << std::ldexp(1.0, -1025); - v << std::ldexp(1.0, 503); - DiagonalPlusLowRank<double> A(d, u, v); - VERIFY_IS_EQUAL(A.capacitance()(0, 0), 2.0); - VERIFY_IS_EQUAL(A.determinant(), std::ldexp(1.0, -521)); - Vec b(1); - b << 1.0; - VERIFY_IS_EQUAL(A.solve(b)(0), std::ldexp(1.0, 521)); - } - { - // Symmetric mirror (U huge, V tiny): the plain capacitance association is - // safe here, but the inverse's D^{-1} U product overflows spuriously while - // the inverse factor -1e400 / (1 + 1e200) ~ -1e200 is representable. - Vec d(1), u(1), v(1); - d << 1e-200; - u << 1e200; - v << 1e-200; - DiagonalPlusLowRank<double> A(d, u, v); - VERIFY(numext::abs(A.determinant() - 1.0) <= 8.0 * eps); - DiagonalPlusLowRank<double> Ainv = A.inverse(); - VERIFY(Ainv.factorU().allFinite()); - VERIFY_IS_APPROX(Ainv.factorU()(0, 0), -1e200); - Vec b(1); - b << 1.0; - VERIFY(A.solve(b).allFinite()); - } - { - // Exact-power-of-two mirror: d = 2^-522, U = 2^502, V = 2^-1025 represent - // A = [1.5 * 2^-522]; D^{-1} U = 2^1024 overflows, yet the inverse factor - // -2^1024 / 1.5 is representable and the solve is exact to rounding. - Vec d(1), u(1), v(1); - d << std::ldexp(1.0, -522); - u << std::ldexp(1.0, 502); - v << std::ldexp(1.0, -1025); - DiagonalPlusLowRank<double> A(d, u, v); - VERIFY_IS_EQUAL(A.capacitance()(0, 0), 1.5); - VERIFY_IS_EQUAL(A.determinant(), 1.5 * std::ldexp(1.0, -522)); - DiagonalPlusLowRank<double> Ainv = A.inverse(); - VERIFY_IS_APPROX(Ainv.factorU()(0, 0), -std::ldexp(1.0 / 1.5, 1024)); - Vec b(1); - b << 3.0; - VERIFY_IS_EQUAL(A.solve(b)(0), std::ldexp(1.0, 523)); - } - { - // Complex analogue of the reviewer's case: the operator is essentially - // [i], the capacitance 1 + 1e200 is representable, and the balancing and - // rescaling act on component magnitudes. - CVec d(1), u(1), v(1); - d << Complex(0, 1e-200); - u << Complex(1e-200, 0); - v << Complex(0, -1e200); - DiagonalPlusLowRank<Complex> A(d, u, v); - VERIFY((numext::isfinite)(numext::real(A.capacitance()(0, 0)))); - VERIFY_IS_APPROX(A.capacitance()(0, 0), Complex(1e200, 0)); - VERIFY_IS_APPROX(A.determinant(), Complex(0, 1)); - } -} - -// Moderate data must keep the plain product association: the normalized -// Woodbury kernels only engage when a conservative exponent bound says a -// product could overflow, so for well-scaled operators the capacitance and the -// solve are bit-identical to the unnormalized evaluation. +// The capacitance and the solve are bit-identical to the plain Woodbury +// evaluation. template <typename Scalar> -void test_dplr_capacitance_fast_path(Index n, Index k) { +void test_dplr_capacitance_plain(Index n, Index k) { typedef Matrix<Scalar, Dynamic, 1> Vec; typedef Matrix<Scalar, Dynamic, Dynamic> Mat; @@ -673,11 +567,10 @@ CALL_SUBTEST_4((test_dplr_delayed_product<std::complex<double>>(10, 3))); CALL_SUBTEST_4((test_dplr_mixed_scalar<double>(12, 3))); CALL_SUBTEST_4((test_dplr_mixed_scalar<float>(9, 2))); - CALL_SUBTEST_4(test_dplr_determinant_scaled()); - CALL_SUBTEST_4(test_dplr_determinant_capacitance()); - CALL_SUBTEST_4(test_dplr_capacitance_overflow()); - CALL_SUBTEST_4((test_dplr_capacitance_fast_path<double>(11, 3))); - CALL_SUBTEST_4((test_dplr_capacitance_fast_path<std::complex<double>>(9, 2))); + CALL_SUBTEST_4(test_dplr_determinant_scaled<>()); + CALL_SUBTEST_4(test_dplr_determinant_capacitance<>()); + CALL_SUBTEST_4((test_dplr_capacitance_plain<double>(11, 3))); + CALL_SUBTEST_4((test_dplr_capacitance_plain<std::complex<double>>(9, 2))); CALL_SUBTEST_4((test_dplr_fixed_rank0<double, 5>())); CALL_SUBTEST_4((test_dplr_fixed_rank0<std::complex<float>, 4>())); }
diff --git a/contrib/test/structured_dpr1.cpp b/contrib/test/structured_dpr1.cpp index 555b978..7150f13 100644 --- a/contrib/test/structured_dpr1.cpp +++ b/contrib/test/structured_dpr1.cpp
@@ -98,6 +98,41 @@ DPR1EigenSolver<Scalar> es(dc, Scalar(1), z); VERIFY(numext::abs(es.eigenvalues()[n - 1] - (Scalar(3) + z.squaredNorm())) <= Scalar(100) * Scalar(n) * NumTraits<Scalar>::epsilon() * (Scalar(3) + z.squaredNorm())); + // Deflating equal poles leaves the pole itself: c has multiplicity n - 1 exactly, for either + // sign of rho (rho < 0 puts c + rho*||z||^2 first). + for (Index i = 0; i + 1 < n; ++i) VERIFY_IS_EQUAL(es.eigenvalues()[i], Scalar(3)); + DPR1EigenSolver<Scalar> esn(dc, Scalar(-1), z, EigenvaluesOnly); + for (Index i = 1; i < n; ++i) VERIFY_IS_EQUAL(esn.eigenvalues()[i], Scalar(3)); + + // Exact triples among distinct poles: a pole of multiplicity k stays an eigenvalue >= k - 1 times. + Vec dt(n); + for (Index i = 0; i < n; ++i) { + const Index pole = 1 + i / 3; + dt[i] = Scalar(pole); + } + check_dpr1<Scalar>(dt, Scalar(2), z); + DPR1EigenSolver<Scalar> est(dt, Scalar(2), z, EigenvaluesOnly); + for (Index i = 0; i < n; i += 3) { + const Index multiplicity = numext::mini<Index>(3, n - i); + VERIFY((est.eigenvalues().array() == dt[i]).count() >= multiplicity - 1); + } +} + +// A Givens deflation with c = O(eps), s ~ 1 across a wide gap: rho*|z_2| = 10.8 eps and +// |c s (d_2 - d_1)| ~ 3 eps bracket tol = 7.2 eps. The secular equation puts an eigenvalue at +// d_2 - O(eps^2), which the deflation must form as a correction to d_2, not to the far pole d_1: +// d_1 + s^2 (d_2 - d_1) inherits the rounding of d_2 - d_1, up to eps/8 >> eps * d_2. +template <typename Scalar> +void test_dpr1_skewed_deflation() { + using Vec = Matrix<Scalar, Dynamic, 1>; + const Scalar eps = NumTraits<Scalar>::epsilon(); + const Scalar small = numext::sqrt(eps) / Scalar(3); // not a power of two, so d_2 - d_1 rounds + Vec d(2), z(2); + d << Scalar(-0.25), small; + z << Scalar(1), Scalar(12) * eps; + check_dpr1<Scalar>(d, Scalar(0.9), z); + DPR1EigenSolver<Scalar> es(d, Scalar(0.9), z, EigenvaluesOnly); + VERIFY(numext::abs(es.eigenvalues()[0] - small) <= Scalar(4) * eps * small); } // Zero and tiny z entries exercise the z-deflation path. @@ -192,6 +227,7 @@ // stableNorm keeps huge z representable: with float z ~ 2e19, z.squaredNorm() // would overflow but rho*||z||^2 is a perfectly ordinary number. +template <typename = void> void test_dpr1_huge_z() { typedef Matrix<float, Dynamic, 1> Vec; Vec d(1), z(1); @@ -209,9 +245,7 @@ VERIFY(numext::abs(es.eigenvalues()[0] - expected) <= tol * expected); // A spectrum that genuinely overflows (top eigenvalue ~1e320 here) must be - // reported, not silently deflated. Note rho*||z||^2 overflowing is *not* by - // itself invalid (see test_dpr1_overflowing_update_representable_spectrum); - // this input is rejected because the eigenvalue itself is not representable. + // reported, not silently deflated. DPR1EigenSolver<double> ov((Matrix<double, Dynamic, 1>(1) << 1.0).finished(), 1e200, (Matrix<double, Dynamic, 1>(1) << 1e60).finished()); VERIFY(ov.info() == InvalidInput); @@ -219,6 +253,7 @@ // Non-finite input must be rejected up front (it would otherwise break the // sorting comparator and silently deflate everything). +template <typename = void> void test_dpr1_nonfinite() { typedef Matrix<double, Dynamic, 1> Vec; Vec d = Vec::Random(6), z = Vec::Random(6); @@ -240,6 +275,7 @@ VERIFY(e5.info() == InvalidInput); } +template <typename = void> void test_dpr1_invalid_options() { typedef Matrix<double, Dynamic, 1> Vec; const Vec d = Vec::LinSpaced(3, 1.0, 3.0); @@ -249,74 +285,11 @@ VERIFY_RAISES_ASSERT((es.compute(d, 1.0, z, ComputeFullU))); } -// Regression (MR 2694 review): poles spanning the full exponent range with a huge but representable -// update. Pole differences (d_1 - d_0 = 1.5*DBL_MAX) overflow unless the problem is rescaled by an -// exact power of two; the solver used to report Success with an infinite eigenvalue. The reference is -// a dense solve of the rescaled problem 2^-e * (d, rho), whose spectrum is exactly 2^-e times the -// original. -void test_dpr1_full_range_poles() { - typedef Matrix<double, Dynamic, 1> Vec; - typedef Matrix<double, Dynamic, Dynamic> Mat; - const double dmax = (std::numeric_limits<double>::max)(); - Vec d(2), z(2); - d << -0.75 * dmax, 0.75 * dmax; - z << 1.0, 1.0; - const double rho = 1e300; - DPR1EigenSolver<double> es(d, rho, z); - VERIFY(es.info() == Success); - VERIFY(es.eigenvalues().allFinite()); - - const int e = 1024; // 2^-e maps 0.75*DBL_MAX into [0.5, 1) - Vec dS(2); - dS << std::ldexp(d[0], -e), std::ldexp(d[1], -e); - const double rhoS = std::ldexp(rho, -e); - Mat dense = Mat(dS.asDiagonal()) + rhoS * z * z.transpose(); - SelfAdjointEigenSolver<Mat> ref(dense, EigenvaluesOnly); - const double scaleS = dS.cwiseAbs().maxCoeff() + rhoS * z.squaredNorm(); - const double tol = 100.0 * NumTraits<double>::epsilon(); - Vec lambdaS(2); - lambdaS << std::ldexp(es.eigenvalues()[0], -e), std::ldexp(es.eigenvalues()[1], -e); - VERIFY((lambdaS - ref.eigenvalues()).cwiseAbs().maxCoeff() <= tol * scaleS); - // Eigenvectors are scale-invariant: check them on the rescaled matrix. - const Mat& V = es.eigenvectors(); - VERIFY((dense * V - V * lambdaS.asDiagonal()).norm() <= tol * scaleS); - VERIFY((V.transpose() * V - Mat::Identity(2, 2)).norm() <= tol); -} - -// Regression (MR 2694 review): z entries at the top of the range make ||z|| -// itself overflow, but the update rho*z*z^T (rank-one eigenvalue ~6.46e306) is -// perfectly representable; the solver used to return InvalidInput. It must -// instead normalize z by an exact power of two, 2^-p z with rho 2^(2p), which -// leaves the matrix identical -- the same rewrite gives the dense reference. -void test_dpr1_huge_z_norm() { - if (!dpr1_preserves_subnormal_inputs<double>()) return; - - typedef Matrix<double, Dynamic, 1> Vec; - typedef Matrix<double, Dynamic, Dynamic> Mat; - const double dmax = (std::numeric_limits<double>::max)(); - Vec d(2), z(2); - d << 0.0, 0.0; - z << dmax, dmax; - const double rho = 1e-310; // subnormal, so rho * 2^(2p) stays finite - DPR1EigenSolver<double> es(d, rho, z); - VERIFY(es.info() == Success); - - const int p = 1024; // 2^-p maps DBL_MAX into [0.5, 1) - Vec zS(2); - zS << std::ldexp(z[0], -p), std::ldexp(z[1], -p); - const double rhoS = std::ldexp(rho, 2 * p); // exact: subnormal scaled up - Mat dense = Mat(d.asDiagonal()) + rhoS * zS * zS.transpose(); - SelfAdjointEigenSolver<Mat> ref(dense, EigenvaluesOnly); - const double scale = rhoS * zS.squaredNorm(); // the rank-one eigenvalue, ~6.46e306 - const double tol = 100.0 * NumTraits<double>::epsilon(); - VERIFY((es.eigenvalues() - ref.eigenvalues()).cwiseAbs().maxCoeff() <= tol * scale); - VERIFY(numext::abs(es.eigenvalues()[1] - scale) <= tol * scale); -} - // Regression (MR 2694 review): a z entry just above the deflation threshold // puts the secular root ~rho*z_0^2 ~ 6.3e-30 from its pole; bisecting the O(1) // bracket down to it takes ~150 halvings, beyond the old 2*digits+32 iteration // cap, and used to report NoConvergence (check_dpr1 requires Success). +template <typename = void> void test_dpr1_near_deflation_root() { typedef Matrix<double, Dynamic, 1> Vec; Vec d(2), z(2); @@ -325,9 +298,10 @@ check_dpr1<double>(d, 1.0, z); } -// Regression (MR 2694 review): finite 1x1 input whose only eigenvalue -// d + rho*z^2 = 2*DBL_MAX overflows. The documented contract for a spectrum -// that is not representable is InvalidInput, not Success carrying infinity. +// Finite 1x1 input whose only eigenvalue d + rho*z^2 = 2*DBL_MAX overflows: the +// contract for a non-finite computed eigenvalue is InvalidInput with NaN +// eigenvalues, not Success carrying infinity. +template <typename = void> void test_dpr1_overflowing_spectrum() { typedef Matrix<double, Dynamic, 1> Vec; const double dmax = (std::numeric_limits<double>::max)(); @@ -337,157 +311,34 @@ DPR1EigenSolver<double> es(d, dmax, z); VERIFY(es.info() == InvalidInput); VERIFY((numext::isnan)(es.eigenvalues()[0])); - // Same with the overflow amplified through z (eigenvalue = 5*DBL_MAX): the - // internal rescaling saturates the unrepresentable eigenvalue to infinity, - // which the contract maps to InvalidInput + NaN rather than Success + inf. + // Same with the overflow amplified through z (eigenvalue = 5*DBL_MAX), where + // rho*||z||^2 itself overflows. z << 2.0; DPR1EigenSolver<double> es2(d, dmax, z); VERIFY(es2.info() == InvalidInput); VERIFY((numext::isnan)(es2.eigenvalues()[0])); + + // An overflowing rho*||z||^2 is reported even when the spectrum {-1e308, 1.5e308} + // is representable, rather than deflated away into diag(d) with Success. + Vec d2(2), z2(2); + d2 << -1.5e308, 0.0; + z2 << 1e154, 1e154; + DPR1EigenSolver<double> es3(d2, 1.0, z2); + VERIFY(es3.info() == InvalidInput); + + // rho == 0 leaves diag(d) whatever the size of z. + Vec d3(2), z3(2); + d3 << 1.0, 2.0; + z3 << 1e308, 1e308; + DPR1EigenSolver<double> es4(d3, 0.0, z3); + VERIFY(es4.info() == Success); + VERIFY_IS_EQUAL(es4.eigenvalues(), d3); } -// Regression (MR 2694 review): the normalized secular problem can round a root across the -// maximum-finite boundary before it is scaled back, so that root's finiteness cannot classify the -// exact spectrum. The first update has lambda = DBL_MAX - 19/256 ulp, which rounds to DBL_MAX; the -// second has lambda = DBL_MAX + 3739/2048 ulp and genuinely overflows. Constants are assembled from -// their exact significands, hexadecimal float literals not being in the C++14 baseline. -void test_dpr1_maximum_range_rounding() { - typedef Matrix<double, Dynamic, 1> Vec; - const double dmax = (std::numeric_limits<double>::max)(); - Vec d(1), z(1); - - d << -dmax; - z << 31.0 / 8.0; - const double rhoRepresentable = std::ldexp(4798840810018093.0, 969); - DPR1EigenSolver<double> representable(d, rhoRepresentable, z); - VERIFY(representable.info() == Success); - VERIFY_IS_EQUAL(representable.eigenvalues()[0], dmax); - - d << dmax; - DPR1EigenSolver<double> representableNegative(d, -rhoRepresentable, z); - VERIFY(representableNegative.info() == Success); - VERIFY_IS_EQUAL(representableNegative.eigenvalues()[0], -dmax); - - d << -dmax; - z << 65.0 / 2.0; - const double rhoOverflow = std::ldexp(8732186543767835.0, 962); - DPR1EigenSolver<double> overflow(d, rhoOverflow, z); - VERIFY(overflow.info() == InvalidInput); - VERIFY((numext::isnan)(overflow.eigenvalues()[0])); - - d << dmax; - DPR1EigenSolver<double> overflowNegative(d, -rhoOverflow, z); - VERIFY(overflowNegative.info() == InvalidInput); - VERIFY((numext::isnan)(overflowNegative.eigenvalues()[0])); -} - -// Multiple exactly representable endpoint terms must be certifiable together: -// rho*z_i^2/(max-d_i) = 1/2 for each i, so the spectrum is exactly {0, max}. -template <typename Scalar> -void test_dpr1_exact_endpoint_sum() { - typedef Matrix<Scalar, Dynamic, 1> Vec; - using std::nextafter; - const Scalar highest = (std::numeric_limits<Scalar>::max)(); - const Scalar boundaryRho = highest / Scalar(2); - const Vec d = Vec::Zero(2); - const Vec z = Vec::Ones(2); - - DPR1EigenSolver<Scalar> positive(d, boundaryRho, z); - VERIFY(positive.info() == Success); - VERIFY_IS_EQUAL(positive.eigenvalues()[0], Scalar(0)); - VERIFY_IS_EQUAL(positive.eigenvalues()[1], highest); - - DPR1EigenSolver<Scalar> negative(d, -boundaryRho, z); - VERIFY(negative.info() == Success); - VERIFY_IS_EQUAL(negative.eigenvalues()[0], -highest); - VERIFY_IS_EQUAL(negative.eigenvalues()[1], Scalar(0)); - - const Vec d4 = Vec::Zero(4); - const Vec z4 = Vec::Ones(4); - DPR1EigenSolver<Scalar> fourTerms(d4, highest / Scalar(4), z4); - VERIFY(fourTerms.info() == Success); - VERIFY_IS_EQUAL(fourTerms.eigenvalues()[3], highest); - - DPR1EigenSolver<Scalar> below(d, nextafter(boundaryRho, Scalar(0)), z); - VERIFY(below.info() == Success); - VERIFY(below.eigenvalues().allFinite()); - - DPR1EigenSolver<Scalar> above(d, nextafter(boundaryRho, highest), z); - VERIFY(above.info() == InvalidInput); - - // The exact top eigenvalue is highest + min. Normalizing the endpoint - // denominator drops that tiny difference, so this case must remain - // conservatively unresolved instead of reusing the exact-equality path. - const Vec minDiagonal = Vec::Constant(2, (std::numeric_limits<Scalar>::min)()); - DPR1EigenSolver<Scalar> unresolved(minDiagonal, boundaryRho, z); - VERIFY(unresolved.info() != Success); -} - -// Regression (MR 2694 review): rho*||z||^2 = 2*max overflows, but the single eigenvalue -// d + rho*z^2 = -max + 2*max = max is representable. The rescaling must come from component exponents -// alone, never materializing rho*||z||^2 at full scale. The secular root sits exactly on the far end -// of its bisection bracket, so the computed eigenvalue lands within a couple of ulps of max rather -// than exactly at it. -template <typename Scalar> -void test_dpr1_overflowing_update_representable_spectrum() { - typedef Matrix<Scalar, Dynamic, 1> Vec; - const Scalar dmax = (std::numeric_limits<Scalar>::max)(); - const Scalar eps = NumTraits<Scalar>::epsilon(); - Vec d(1), z(1); - d << -dmax; - z << Scalar(2); - DPR1EigenSolver<Scalar> es(d, dmax / Scalar(2), z); - VERIFY(es.info() == Success); - const Scalar lam = es.eigenvalues()[0]; - VERIFY((numext::isfinite)(lam)); - VERIFY(numext::abs(lam - dmax) <= Scalar(4) * eps * dmax); - // n = 1: the eigenvector is the coordinate axis, exactly (up to sign). - VERIFY(numext::abs(es.eigenvectors()(0, 0)) == Scalar(1)); -} - -// Regression (MR 2694 review), n > 1 version: poles near -0.75*DBL_MAX with -// rho*||z||^2 ~ 1.2*DBL_MAX (overflowing if materialized) but every eigenvalue -// representable -- the top root lands near +0.4*DBL_MAX, the rest interlace -// the poles. Verified against a dense solve of the exactly rescaled problem, -// whose spectrum is exactly 2^-e times the original. -void test_dpr1_huge_representable_spectrum() { - typedef Matrix<double, Dynamic, 1> Vec; - typedef Matrix<double, Dynamic, Dynamic> Mat; - const Index n = 8; - const double dmax = (std::numeric_limits<double>::max)(); - Vec d(n), z(n); - for (Index i = 0; i < n; ++i) { - d[i] = -dmax * (0.70 + 0.01 * double(i)); - z[i] = 1.0 + double(i) / 8.0; // ||z||^2 ~ 18.6 - } - const double rho = dmax / 16; - DPR1EigenSolver<double> es(d, rho, z); - VERIFY(es.info() == Success); - VERIFY(es.eigenvalues().allFinite()); - - const int e = 1025; // 2^-e maps every |d_i| and rho*||z||^2 below 1 - Vec dS(n), lambdaS(n); - for (Index i = 0; i < n; ++i) { - dS[i] = std::ldexp(d[i], -e); - lambdaS[i] = std::ldexp(es.eigenvalues()[i], -e); - } - const double rhoS = std::ldexp(rho, -e); - Mat dense = Mat(dS.asDiagonal()) + rhoS * z * z.transpose(); - SelfAdjointEigenSolver<Mat> ref(dense, EigenvaluesOnly); - const double scaleS = dS.cwiseAbs().maxCoeff() + rhoS * z.squaredNorm(); - const double tol = 100.0 * double(n) * NumTraits<double>::epsilon(); - VERIFY((lambdaS - ref.eigenvalues()).cwiseAbs().maxCoeff() <= tol * scaleS); - // Eigenvectors are scale-invariant: check them on the rescaled matrix. - const Mat& V = es.eigenvectors(); - VERIFY((dense * V - V * lambdaS.asDiagonal()).norm() <= tol * scaleS * numext::sqrt(double(n))); - VERIFY((V.transpose() * V - Mat::Identity(n, n)).norm() <= tol); -} - -// Mirror case at the bottom of the range: d and rho near DBL_MIN. The same -// exact power-of-two machinery scales *up* (scaleExp < 0), which needs no -// separate underflow handling -- scaling up from anywhere at or above the -// normal range is exact, and the eigenvalues here are all normal. Verified -// against a dense solve of the exactly upscaled problem. +// d and rho near DBL_MIN: the problem scaling scales *up* (scaleExp < 0), which +// is exact from anywhere at or above the normal range, and the eigenvalues here +// are all normal. Verified against a dense solve of the exactly upscaled problem. +template <typename = void> void test_dpr1_tiny_scale() { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -531,22 +382,9 @@ } } -void test_dpr1_ftz_mode() { - const double underflowBefore = underflowProbe<double>(); - bool flushToZeroSupported = false; - { - ScopedFlushToZero flushToZero; - flushToZeroSupported = flushToZero.isSupported(); - if (flushToZeroSupported) { - VERIFY_IS_EQUAL(underflowProbe<double>(), 0.0); - test_dpr1_edges<double>(); - } - } - if (flushToZeroSupported) VERIFY_IS_EQUAL(underflowProbe<double>(), underflowBefore); -} - // A huge diagonal spread makes every z entry individually negligible even though // rho itself is not: the whole update deflates and m == 0. +template <typename = void> void test_dpr1_all_deflated() { typedef Matrix<double, Dynamic, 1> Vec; Vec d(4), z(4); @@ -576,6 +414,9 @@ CALL_SUBTEST_2((test_dpr1_rank_one<float>(11))); CALL_SUBTEST_2((test_dpr1_clustered<long double>(12))); CALL_SUBTEST_2((test_dpr1_sparse_z<long double>(9))); + CALL_SUBTEST_2((test_dpr1_skewed_deflation<float>())); + CALL_SUBTEST_2((test_dpr1_skewed_deflation<double>())); + CALL_SUBTEST_2((test_dpr1_skewed_deflation<long double>())); CALL_SUBTEST_3((test_dpr1_edges<double>())); CALL_SUBTEST_3((test_dpr1_edges<float>())); @@ -583,7 +424,7 @@ CALL_SUBTEST_3((test_dpr1_scaling<double>(14))); CALL_SUBTEST_3((test_dpr1_edges<long double>())); CALL_SUBTEST_3((test_dpr1_scaling<long double>(14))); - CALL_SUBTEST_3(test_dpr1_invalid_options()); + CALL_SUBTEST_3(test_dpr1_invalid_options<>()); // Gu-Eisenstat litmus (close but undeflated poles), orthogonality growth at // larger n, extreme scales, non-finite rejection, full-deflation pin. @@ -591,24 +432,13 @@ CALL_SUBTEST_4((test_dpr1_close_undeflated<float>(16))); CALL_SUBTEST_4((test_dpr1_random<double>(128))); CALL_SUBTEST_4((test_dpr1_close_undeflated<long double>(16))); - CALL_SUBTEST_4(test_dpr1_huge_z()); - CALL_SUBTEST_4(test_dpr1_nonfinite()); - CALL_SUBTEST_4(test_dpr1_all_deflated()); + CALL_SUBTEST_4(test_dpr1_huge_z<>()); + CALL_SUBTEST_4(test_dpr1_nonfinite<>()); + CALL_SUBTEST_4(test_dpr1_all_deflated<>()); - // MR 2694 review regressions: exponent-range extremes and deep bisection. - CALL_SUBTEST_5(test_dpr1_full_range_poles()); - CALL_SUBTEST_5(test_dpr1_huge_z_norm()); - CALL_SUBTEST_5(test_dpr1_near_deflation_root()); - CALL_SUBTEST_5(test_dpr1_overflowing_spectrum()); - CALL_SUBTEST_5(test_dpr1_maximum_range_rounding()); - CALL_SUBTEST_5((test_dpr1_exact_endpoint_sum<float>())); - CALL_SUBTEST_5((test_dpr1_exact_endpoint_sum<double>())); - CALL_SUBTEST_5((test_dpr1_exact_endpoint_sum<long double>())); - CALL_SUBTEST_5((test_dpr1_overflowing_update_representable_spectrum<double>())); - CALL_SUBTEST_5((test_dpr1_overflowing_update_representable_spectrum<float>())); - CALL_SUBTEST_5((test_dpr1_overflowing_update_representable_spectrum<long double>())); - CALL_SUBTEST_5(test_dpr1_huge_representable_spectrum()); - CALL_SUBTEST_5(test_dpr1_tiny_scale()); - CALL_SUBTEST_5(test_dpr1_ftz_mode()); + // Deep bisection, an overflowing spectrum and the bottom of the range. + CALL_SUBTEST_5(test_dpr1_near_deflation_root<>()); + CALL_SUBTEST_5(test_dpr1_overflowing_spectrum<>()); + CALL_SUBTEST_5(test_dpr1_tiny_scale<>()); } }
diff --git a/contrib/test/structured_kronecker.cpp b/contrib/test/structured_kronecker.cpp index 4b2b77c..be59af9 100644 --- a/contrib/test/structured_kronecker.cpp +++ b/contrib/test/structured_kronecker.cpp
@@ -220,6 +220,114 @@ VERIFY_IS_APPROX(z, (denseC * xr).eval()); } +// Products and solves stack right-hand sides in chunks sized by the L2 cache +// size. A tiny L2 forces chunks of one or two columns with a short last chunk, +// and row-major operands are read and written through strided columns. +template <typename Lhs, typename Rhs> +void check_kron_chunked_product(const KroneckerOperator<Lhs, Rhs>& K) { + using Scalar = typename Lhs::Scalar; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using RowMat = Matrix<Scalar, Dynamic, Dynamic, RowMajor>; + const Mat dense = K; + const RowMat X = RowMat::Random(K.cols(), 5), Y0 = RowMat::Random(K.rows(), 5); + const RowMat expected = dense * X; + RowMat Y = K * X; + VERIFY_IS_APPROX(Y, expected); + Y = Y0; + Y.noalias() += K * X; + VERIFY_IS_APPROX(Y, (Y0 + expected).eval()); + Y = Y0; + Y.noalias() -= K * X; + VERIFY_IS_APPROX(Y, (Y0 - expected).eval()); + VERIFY_IS_APPROX(Mat(K * Mat(X)), Mat(expected)); +} + +template <typename Scalar> +void test_kron_chunked_products() { + using Vec = Matrix<Scalar, Dynamic, 1>; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using Sparse = SparseMatrix<Scalar>; + using Diag = DiagonalMatrix<Scalar, Dynamic>; + const std::ptrdiff_t l1 = l1CacheSize(), l2 = l2CacheSize(), l3 = l3CacheSize(); + // A budget of 32 stacked entries: 2 columns of the 16-entry workspace of + // 3x2 (x) 2x3, chunks of 2, 2, 1 for 5 right-hand sides. + setCpuCacheSizes(l1, std::ptrdiff_t(8 * 32 * sizeof(Scalar)), l3); + + const Mat A = Mat::Random(3, 2), B = Mat::Random(2, 3); + const Sparse As = A.sparseView(), Bs = B.sparseView(); + check_kron_chunked_product(makeKroneckerOperator(A, B)); + check_kron_chunked_product(makeKroneckerOperator(As, Bs)); + check_kron_chunked_product(makeKroneckerOperator(A, Diag(Vec::Random(3)))); + check_kron_chunked_product(makeKroneckerOperator(Diag(Vec::Random(2)), Bs)); + check_kron_chunked_product(makeKroneckerOperator(Mat::Identity(2, 2), Bs)); + check_kron_chunked_product(makeKroneckerOperator(A, Mat::Identity(3, 3))); + check_kron_chunked_product(makeKroneckerOperator(Mat::Identity(2, 3), B)); + check_kron_chunked_product(makeKroneckerOperator(Mat::Identity(2, 2), Mat::Identity(2, 2))); + check_kron_chunked_product(makeKroneckerOperator(Mat::Random(1, 2), As, Mat::Random(2, 1))); + + setCpuCacheSizes(l1, l2, l3); +} + +template <typename Scalar> +void test_kron_chunked_solves() { + using RealScalar = typename NumTraits<Scalar>::Real; + using Vec = Matrix<Scalar, Dynamic, 1>; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using RowMat = Matrix<Scalar, Dynamic, Dynamic, RowMajor>; + using Sparse = SparseMatrix<Scalar>; + using Diag = DiagonalMatrix<Scalar, Dynamic>; + const std::ptrdiff_t l1 = l1CacheSize(), l2 = l2CacheSize(), l3 = l3CacheSize(); + // A budget of 32 stacked entries: 2 columns of the 16-entry workspace of + // 3x2 (x) 2x3, chunks of 2, 2, 1 for 5 right-hand sides. + setCpuCacheSizes(l1, std::ptrdiff_t(8 * 32 * sizeof(Scalar)), l3); + + const RowMat b = RowMat::Random(4, 5); + const Mat Ad = Mat::Random(2, 2) + RealScalar(4) * Mat::Identity(2, 2); + const Mat Bd = Mat::Random(2, 2) + RealScalar(4) * Mat::Identity(2, 2); + const KroneckerOperator<Mat, Mat> Ks(Ad, Bd); + const Mat x = Mat(Ks).partialPivLu().solve(Mat(b)); + VERIFY_IS_APPROX(Ks.solve(b), x); + const KroneckerOperator<Sparse, Mat> Kss(Ad.sparseView(), Bd); + VERIFY_IS_APPROX(Kss.solve(b), x); + const KroneckerOperator<Mat, Mat> Kl(Mat::Random(2, 1), Mat::Random(2, 2)); + VERIFY_IS_APPROX(Kl.leastSquaresSolve(b), Mat(Mat(Kl).completeOrthogonalDecomposition().solve(Mat(b)))); + // A diagonal factor solves by entrywise division of the stacked right-hand sides. + const Vec d = Vec::Random(2) + Vec::Constant(2, Scalar(4)); + const KroneckerOperator<Diag, Mat> Kdl(Diag(d), Bd); + VERIFY_IS_APPROX(Kdl.solve(b), Mat(Mat(Kdl).partialPivLu().solve(Mat(b)))); + const KroneckerOperator<Mat, Diag> Kdr(Ad, Diag(d)); + VERIFY_IS_APPROX(Kdr.solve(b), Mat(Mat(Kdr).partialPivLu().solve(Mat(b)))); + const auto Kil = makeKroneckerOperator(Mat::Identity(2, 2), Bd); + VERIFY_IS_APPROX(Kil.solve(b), Mat(Mat(Kil).partialPivLu().solve(Mat(b)))); + const auto Kn = makeKroneckerOperator(Ad, Bd, Mat::Identity(1, 1)); + VERIFY_IS_APPROX(Kn.solve(b), x); + + setCpuCacheSizes(l1, l2, l3); +} + +// A real operator applied to complex right-hand sides, in several chunks. +template <typename RealScalar> +void test_kron_chunked_mixed_scalar() { + using Complex = std::complex<RealScalar>; + using RMat = Matrix<RealScalar, Dynamic, Dynamic>; + using CMat = Matrix<Complex, Dynamic, Dynamic>; + using CRowMat = Matrix<Complex, Dynamic, Dynamic, RowMajor>; + const std::ptrdiff_t l1 = l1CacheSize(), l2 = l2CacheSize(), l3 = l3CacheSize(); + setCpuCacheSizes(l1, std::ptrdiff_t(8 * 32 * sizeof(Complex)), l3); + + const KroneckerOperator<RMat, RMat> K(RMat::Random(3, 2), RMat::Random(2, 3)); + const CMat dense = RMat(K).template cast<Complex>(); + const CRowMat X = CRowMat::Random(K.cols(), 5); + CRowMat Y = K * X; + VERIFY_IS_APPROX(Y, CRowMat(dense * X)); + const CRowMat Y0 = CRowMat::Random(K.rows(), 5); + Y = Y0; + Y.noalias() += K * X; + VERIFY_IS_APPROX(Y, CRowMat(Y0 + dense * X)); + + setCpuCacheSizes(l1, l2, l3); +} + template <typename Scalar> void test_kron_solve(Index n1, Index n2) { typedef typename NumTraits<Scalar>::Real RealScalar; @@ -276,35 +384,62 @@ VERIFY_IS_APPROX(x, dense.completeOrthogonalDecomposition().solve(b).eval()); } -// Near-rank-deficiency must be judged at the product level: with +// leastSquaresSolve() decomposes each factor scaled by an exact power of two, +// so scaling a factor by 2^s scales the solution by exactly 2^-s -- including +// for s where the unscaled pivoted QR's squared column norms overflow +// (2^(2s) > max) or underflow. A non-finite factor solves to NaN +// instead of being ranked out by the pivoted QR. +template <typename Scalar> +void test_kron_least_squares_scaling(Index m1, Index n1, Index m2, Index n2) { + typedef Matrix<Scalar, Dynamic, Dynamic> Mat; + typedef typename NumTraits<Scalar>::Real RealScalar; + const int maxExp = std::numeric_limits<RealScalar>::max_exponent; + + Mat A = Mat::Random(m1, n1), B = Mat::Random(m2, n2), b = Mat::Random(m1 * m2, 3); + const Mat x0 = KroneckerOperator<Mat, Mat>(A, B).leastSquaresSolve(b); + for (int s : {maxExp / 2 + 8, -(maxExp / 2 + 12)}) { + const RealScalar up = std::ldexp(RealScalar(1), s), down = std::ldexp(RealScalar(1), -s); + const Mat xa = KroneckerOperator<Mat, Mat>(A * up, B).leastSquaresSolve(b); + const Mat xb = KroneckerOperator<Mat, Mat>(A, B * down).leastSquaresSolve(b); + VERIFY((xa * up).eval() == x0); + VERIFY((xb * down).eval() == x0); + } + + Mat Ainf = A, Bnan = B; + Ainf(m1 - 1, 0) = Scalar(std::numeric_limits<RealScalar>::infinity()); + Bnan(0, n2 - 1) = Scalar(std::numeric_limits<RealScalar>::quiet_NaN()); + const Mat xinf = KroneckerOperator<Mat, Mat>(Ainf, B).leastSquaresSolve(b); + const Mat xnan = KroneckerOperator<Mat, Mat>(A, Bnan).leastSquaresSolve(b); + VERIFY(xinf.array().isNaN().all()); + VERIFY(xnan.array().isNaN().all()); +} + +// rank() judges near-rank-deficiency at the product level: with // A = B = diag(1, 1e-8) each factor is full rank against its own threshold, but // the smallest singular value of the product, 1e-16, falls below the -// product-level threshold min(rows,cols) * eps * sigma_max(A) * sigma_max(B). -// The rank is 3 -- matching the dense complete orthogonal decomposition -- and -// leastSquaresSolve() must truncate the 1e-16 mode instead of inverting it. +// product-level threshold min(rows,cols) * eps * sigma_max(A) * sigma_max(B), +// so rank() is 3 -- matching the dense complete orthogonal decomposition. +// leastSquaresSolve() decides rank per factor instead (it never forms the +// pairwise products), so it applies A^+ (x) B^+ with both 1e-8 modes inverted. void test_kron_product_level_rank() { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; - const double small = 1e-8; // sqrt(eps)-ish: kept per factor, truncated as a pairwise product + const double small = 1e-8; Mat A(2, 2); A << 1, 0, 0, small; Mat B = A; KroneckerOperator<Mat, Mat> K(A, B); Mat dense = reference_kron<double>(A, B); - CompleteOrthogonalDecomposition<Mat> cod(dense); - VERIFY_IS_EQUAL(cod.rank(), 3); - VERIFY_IS_EQUAL(K.rank(), cod.rank()); + VERIFY_IS_EQUAL(K.rank(), 3); + VERIFY_IS_EQUAL(K.rank(), CompleteOrthogonalDecomposition<Mat>(dense).rank()); + Mat Ainv(2, 2); + Ainv << 1, 0, 0, 1 / small; Vec b = Vec::Random(4); Vec x = K.leastSquaresSolve(b); - VERIFY_IS_APPROX(x, cod.solve(b).eval()); - // Direct encoding of "the 1e-16 mode is not inverted": the solution is bounded - // by the reciprocal of the smallest kept singular value (with slack), far below - // the 1e16 blow-up an inverted product mode would produce. - const double invertedModeBound = double(10) / small; // smallest kept product singular value is `small` - VERIFY(x.norm() <= invertedModeBound * b.norm()); + VERIFY_IS_APPROX(x, (reference_kron<double>(Ainv, Ainv) * b).eval()); } // det(A (x) B) = det(A)^n2 * det(B)^n1 must be accumulated with exponent @@ -347,97 +482,6 @@ VERIFY_IS_EQUAL(K4.determinant(), 0.0); } -// Reviewer repro: (A (x) B) with A = [1e-200] and B = [1e200] is exactly the -// identity, but evaluating (B X) A^T without protection overflows in -// B X = 1e400 and returns Inf. The power-of-two pre-scaling must keep every -// intermediate finite whenever the true result is representable -- and powers of -// two shift only exponents, so no accuracy may be lost. -void test_kron_extreme_scale_product() { - typedef Matrix<double, Dynamic, 1> Vec; - typedef Matrix<double, Dynamic, Dynamic> Mat; - - Mat A(1, 1), B(1, 1); - A << 1e-200; - B << 1e200; - KroneckerOperator<Mat, Mat> K(A, B); - Vec x(1); - x << 1e200; - Vec y = K * x; - VERIFY(y.allFinite()); - VERIFY_IS_APPROX(y[0], 1e200); - - // Same magnitudes through solve(): with the factors swapped the operator is - // still the identity, and the intermediate B^{-1} mat(b) = 1e400 overflows; - // the normalized-frame solve must return the exact solution. - KroneckerOperator<Mat, Mat> Ks(B, A); - Vec b(1); - b << 1e200; - Vec xs = Ks.solve(b); - VERIFY(xs.allFinite()); - VERIFY_IS_APPROX(xs[0], 1e200); - - // Identity built from scaled unit matrices: the first GEMM is ~1e400 while - // K * x == x exactly. - Mat A2 = 1e-200 * Mat::Identity(2, 2), B2 = 1e200 * Mat::Identity(3, 3); - KroneckerOperator<Mat, Mat> K2(A2, B2); - Vec x2 = 1e200 * Vec::Random(6); - Vec y2 = K2 * x2; - VERIFY(y2.allFinite()); - VERIFY_IS_APPROX(y2, x2); - - // Multi-dimensional overflow boundary: the intermediate B X ~ 2^1103 overflows - // while the true result ~ 2^904 is representable -- and the dense reference can - // compute it, because the materialized product has moderate entries ~ 2^400. - Mat A3 = std::ldexp(1.0, -200) * Mat::Random(2, 2); - Mat B3 = std::ldexp(1.0, 600) * Mat::Random(2, 2); - KroneckerOperator<Mat, Mat> K3(A3, B3); - Mat dense3 = reference_kron<double>(A3, B3); - Vec x3 = std::ldexp(1.0, 500) * Vec::Random(4); - Vec y3 = K3 * x3; - VERIFY(y3.allFinite()); - VERIFY_IS_APPROX(y3, (dense3 * x3).eval()); - - // A genuinely overflowing result must still saturate to infinity. - KroneckerOperator<Mat, Mat> K4(B, B); // [1e200] (x) [1e200] - Vec y4 = K4 * x; // true result 1e600 - VERIFY((numext::isinf)(y4[0])); - VERIFY(y4[0] > 0); -} - -// Complex analogue: the exponent bounds must come from the component-wise -// magnitudes. A finite complex value near the overflow threshold has a -// non-representable modulus, which would silently disable the scaling (and a -// result with components near the threshold has one too, which must not be -// destroyed on the way back). -void test_kron_extreme_scale_complex() { - typedef std::complex<double> Complex; - typedef Matrix<Complex, Dynamic, 1> Vec; - typedef Matrix<Complex, Dynamic, Dynamic> Mat; - - Mat A(1, 1), B(1, 1); - A << Complex(1e-300, 0.0); - B << Complex(1e308, 1e308); // |B(0,0)| overflows; the components are finite - KroneckerOperator<Mat, Mat> K(A, B); - Vec x(1); - x << Complex(1e300, 0.0); - Vec y = K * x; - VERIFY(y.allFinite()); - // Exact scalar reference, evaluated in an order that never leaves the - // representable range: (A * B) * x = (1e8 + 1e8 i) * 1e300. - const Complex expected = (A(0, 0) * B(0, 0)) * x[0]; - VERIFY_IS_APPROX(y[0], expected); - - Mat A2(1, 1), B2(1, 1); - A2 << Complex(1e-200, 0.0); - B2 << Complex(1.2e200, -1.3e200); - KroneckerOperator<Mat, Mat> K2(A2, B2); - Vec x2(1); - x2 << Complex(1.1e200, -0.7e200); - Vec y2 = K2 * x2; - VERIFY(y2.allFinite()); - VERIFY_IS_APPROX(y2[0], ((A2(0, 0) * B2(0, 0)) * x2[0])); -} - // Reviewer repro: with A = [1e-310] (subnormal) the absolute rank threshold // min(rows,cols) * eps * sa_max underflows to zero before the sb_max // multiplication, so every pairwise singular-value product used to pass and @@ -482,8 +526,7 @@ // The ratio test alone dropped SVDBase's smallest-normal clamp: a subnormal // product singular value, whose reciprocal overflows, must count as an exact // zero. A = [1], B = [DBL_MIN/2] has the singular value DBL_MIN/2: the dense -// rank is 0 and the pseudo-inverse is zero, but without the clamp the -// structured rank was 1 and leastSquaresSolve returned ~9e307. The boundary +// rank is 0, but without the clamp the structured rank was 1. The boundary // DBL_MIN itself stays rank one, matching SVDBase::rank()'s >= convention. void test_kron_rank_min_normal_clamp() { typedef Matrix<double, Dynamic, 1> Vec; @@ -500,8 +543,12 @@ VERIFY_IS_EQUAL(K.rank(), svd.rank()); Vec b(1); b << 1.0; - VERIFY_IS_EQUAL(K.leastSquaresSolve(b).norm(), 0.0); VERIFY_IS_EQUAL(svd.solve(b).norm(), 0.0); + // leastSquaresSolve has no singular values to clamp: the factor is full rank + // against its own scale, and 1 / (DBL_MIN / 2) is representable. + Vec x = K.leastSquaresSolve(b); + VERIFY(x.allFinite()); + VERIFY_IS_APPROX(x[0], 2 / mn); // Boundary: the smallest normal number is the smallest kept singular value. Mat B2(1, 1); @@ -515,69 +562,6 @@ VERIFY_IS_APPROX(x2[0], 1.0 / mn); } -// solve() must run in a normalized frame; the previous detect-and-retry missed -// (a) silent underflow -- the intermediate B^-1 mat(b) flushes to zero, which is -// finite, so nothing triggered a retry -- and (b) factors whose raw inverse is -// not representable even against a unit right-hand side. -void test_kron_solve_normalized() { - typedef Matrix<double, Dynamic, 1> Vec; - typedef Matrix<double, Dynamic, Dynamic> Mat; - - // (a) Identity operator, tiny right-hand side: B^-1 * b = 1e-400 used to - // underflow silently to zero. - for (Index n : {Index(1), Index(5)}) { - Mat A = Mat::Identity(n, n), B = Mat::Identity(n, n); - A *= 1e-200; - B *= 1e200; - KroneckerOperator<Mat, Mat> K(A, B); - Vec b = Vec::Constant(n * n, 1e-200); - Vec x = K.solve(b); - VERIFY_IS_APPROX((x / 1e-200).eval(), Vec::Ones(n * n).eval()); - - // (b) B^-1 ~ 1e310 is not representable, yet the operator is ~0.01 and the - // solution ~100: in the normalized frame every intermediate is bounded by the - // conditioning of the factors. - Mat A2 = Mat::Identity(n, n), B2 = Mat::Identity(n, n); - A2 *= 1e308; - B2 *= 1e-310; // subnormal - KroneckerOperator<Mat, Mat> K2(A2, B2); - Vec b2 = Vec::Ones(n * n); - Vec x2 = K2.solve(b2); - VERIFY(x2.allFinite()); - VERIFY_IS_APPROX(x2, Vec::Constant(n * n, 1.0 / (A2(0, 0) * B2(0, 0))).eval()); - } -} - -// Moderate inputs must be bit-identical to the unnormalized evaluation: partial -// pivoting is invariant under the uniform power-of-two normalization and every -// substitution step scales exactly with it. -template <typename Scalar> -void test_kron_solve_bit_identity(Index n1, Index n2) { - typedef typename NumTraits<Scalar>::Real RealScalar; - typedef Matrix<Scalar, Dynamic, 1> Vec; - typedef Matrix<Scalar, Dynamic, Dynamic> Mat; - // The reference below reproduces the member-internal evaluation, which pins - // its workspaces to column-major storage. - typedef Matrix<Scalar, Dynamic, Dynamic, ColMajor> CmMat; - - Mat A = Mat::Random(n1, n1) + RealScalar(2 * n1) * Mat::Identity(n1, n1); - Mat B = Mat::Random(n2, n2) + RealScalar(2 * n2) * Mat::Identity(n2, n2); - KroneckerOperator<Mat, Mat> K(A, B); - - Mat Bm = Mat::Random(n1 * n2, 2); - Mat X = K.solve(Bm); - - // Raw, unnormalized reference evaluation. - PartialPivLU<CmMat> luA(A), luB(B); - for (Index k = 0; k < Bm.cols(); ++k) { - const Vec bc = Bm.col(k); - const Map<const CmMat> Bmat(bc.data(), n2, n1); - const CmMat Xref = luA.solve(luB.solve(Bmat).transpose()).transpose(); - const Map<const Vec> xref(Xref.data(), Xref.size()); - VERIFY_IS_EQUAL(X.col(k).eval(), Vec(xref)); - } -} - template <typename Scalar> void test_kron_eigen(Index n1, Index n2) { typedef typename NumTraits<Scalar>::Real RealScalar; @@ -918,75 +902,60 @@ // factor decompositions): eigen-residual, SVD reconstruction, rank and // minimum-norm least squares against the dense references. template <typename Scalar> -void test_kron_diag_decompositions(Index n1, Index n2) { - typedef typename NumTraits<Scalar>::Real RealScalar; - typedef std::complex<RealScalar> Complex; - typedef Matrix<Scalar, Dynamic, 1> Vec; - typedef Matrix<Scalar, Dynamic, Dynamic> Mat; - typedef Matrix<Complex, Dynamic, Dynamic> CMat; - typedef Matrix<Complex, Dynamic, 1> CVec; - typedef DiagonalMatrix<Scalar, Dynamic> Diag; +struct DiagKronecker { + using Vec = Matrix<Scalar, Dynamic, 1>; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using Diag = DiagonalMatrix<Scalar, Dynamic>; + DiagKronecker(Index n1, Index n2) + : D(Vec::Random(n1)), B(Mat::Random(n2, n2)), K(D, B), dense(reference_kron<Scalar>(D.toDenseMatrix(), B)) {} + Diag D; + Mat B; + KroneckerOperator<Diag, Mat> K; + Mat dense; +}; - const Diag D(Vec::Random(n1)); - Mat B = Mat::Random(n2, n2); - KroneckerOperator<Diag, Mat> K(D, B); - Mat dense = reference_kron<Scalar>(D.toDenseMatrix(), B); - - CVec lambda = K.eigenvalues(); - CMat V = K.eigenvectors(); - VERIFY_IS_APPROX((dense.template cast<Complex>() * V).eval(), (V * lambda.asDiagonal()).eval()); - - Matrix<RealScalar, Dynamic, 1> sv = K.singularValues(); - Mat U = K.matrixU(), W = K.matrixV(); - VERIFY_IS_APPROX((U * sv.template cast<Scalar>().asDiagonal() * W.adjoint()).eval(), dense); - - VERIFY_IS_EQUAL(K.rank(), dense.completeOrthogonalDecomposition().rank()); - Vec b = Vec::Random(n1 * n2); - VERIFY_IS_APPROX(K.leastSquaresSolve(b), dense.completeOrthogonalDecomposition().solve(b).eval()); +// The decomposition family on a diagonal factor, one test per decomposition to +// keep each part's compile small. +template <typename Scalar> +void test_kron_diag_eigen(Index n1, Index n2) { + using Complex = std::complex<typename NumTraits<Scalar>::Real>; + using CMat = Matrix<Complex, Dynamic, Dynamic>; + using CVec = Matrix<Complex, Dynamic, 1>; + const DiagKronecker<Scalar> f(n1, n2); + const CVec lambda = f.K.eigenvalues(); + const CMat V = f.K.eigenvectors(); + VERIFY_IS_APPROX((f.dense.template cast<Complex>() * V).eval(), (V * lambda.asDiagonal()).eval()); } -// The overflow-hardening contract must hold through diagonal factors: products -// and solves with extreme diagonal magnitudes stay finite whenever the true -// result is representable, and the balanced determinant survives partial -// products that overflow on their own. -void test_kron_diag_extreme_scale() { +template <typename Scalar> +void test_kron_diag_svd(Index n1, Index n2) { + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + const DiagKronecker<Scalar> f(n1, n2); + const Matrix<typename NumTraits<Scalar>::Real, Dynamic, 1> sv = f.K.singularValues(); + const Mat U = f.K.matrixU(), W = f.K.matrixV(); + VERIFY_IS_APPROX((U * sv.template cast<Scalar>().asDiagonal() * W.adjoint()).eval(), f.dense); +} + +template <typename Scalar> +void test_kron_diag_rank(Index n1, Index n2) { + const DiagKronecker<Scalar> f(n1, n2); + VERIFY_IS_EQUAL(f.K.rank(), f.dense.completeOrthogonalDecomposition().rank()); +} + +template <typename Scalar> +void test_kron_diag_least_squares(Index n1, Index n2) { + using Vec = Matrix<Scalar, Dynamic, 1>; + const DiagKronecker<Scalar> f(n1, n2); + const Vec b = Vec::Random(n1 * n2); + VERIFY_IS_APPROX(f.K.leastSquaresSolve(b), f.dense.completeOrthogonalDecomposition().solve(b).eval()); +} + +// Through diagonal factors, the balanced determinant survives partial products +// that overflow on their own. +void test_kron_diag_determinant_scaling() { typedef Matrix<double, Dynamic, 1> Vec; - typedef Matrix<double, Dynamic, Dynamic> Mat; typedef DiagonalMatrix<double, Dynamic> Diag; - // diag(1e-200) (x) [1e200] is the identity; the unprotected application - // B X = 1e400 would overflow. - Diag D(Vec::Constant(1, 1e-200)); - Mat B(1, 1); - B << 1e200; - KroneckerOperator<Diag, Mat> K(D, B); - Vec x(1); - x << 1e200; - Vec y = K * x; - VERIFY(y.allFinite()); - VERIFY_IS_APPROX(y[0], 1e200); - - // Same magnitudes through solve(), diagonal on the divided side. - KroneckerOperator<Mat, Diag> Ks(B, D); - Vec b(1); - b << 1e200; - Vec xs = Ks.solve(b); - VERIFY(xs.allFinite()); - VERIFY_IS_APPROX(xs[0], 1e200); - - // Identity built from extreme diagonal scales: products and the entrywise - // divisions of the diagonal-diagonal solve run in the normalized frame. - Diag Da(Vec::Constant(2, 1e-200)), Db(Vec::Constant(3, 1e200)); - KroneckerOperator<Diag, Diag> K2(Da, Db); - Vec x2 = 1e200 * Vec::Random(6); - Vec y2 = K2 * x2; - VERIFY(y2.allFinite()); - VERIFY_IS_APPROX(y2, x2); - Vec b2 = 1e-200 * Vec::Random(6); - Vec xd = K2.solve(b2); - VERIFY(xd.allFinite()); - VERIFY_IS_APPROX(xd, b2); - // det(diag(1e200))^2 = 1e400 overflows on its own, yet the determinant of // diag(1e200) (x) diag(1e-100, 1e-100) is 1e200 -- and the diagonal path // must balance exactly like the LU path. @@ -1318,8 +1287,8 @@ VERIFY_IS_APPROX((K.adjoint() * y).eval(), (dense.adjoint() * y).eval()); } -// solve() through sparse factors: one SparseLU per sparse factor, then column -// solves in the normalized frame. All factor-kind mixes against the dense LU. +// solve() through sparse factors: one SparseLU per sparse factor, then its +// substitutions. All factor-kind mixes against the dense LU. template <typename Scalar> void test_kron_sparse_solve(Index n1, Index n2) { using RealScalar = typename NumTraits<Scalar>::Real; @@ -1477,29 +1446,6 @@ VERIFY_IS_APPROX(Mat(M), dense); } -// Moderate inputs must be bit-identical to the unnormalized SparseLU -// evaluation: the column ordering is structural, magnitude pivoting is -// invariant under the uniform power-of-two normalization, and every -// substitution scales exactly with it. -template <typename Scalar> -void test_kron_sparse_solve_bit_identity(Index n1, Index n2) { - using Vec = Matrix<Scalar, Dynamic, 1>; - using Mat = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; - using Sparse = SparseMatrix<Scalar>; - - const Sparse S1 = random_sparse_dominant<Scalar>(n1), S2 = random_sparse_dominant<Scalar>(n2); - KroneckerOperator<Sparse, Sparse> K(S1, S2); - Vec b = Vec::Random(n1 * n2); - Vec x = K.solve(b); - - SparseLU<Sparse> luA(S1), luB(S2); - const Mat Bm = b.reshaped(n2, n1); - const Mat Y = luB.solve(Bm); // S2^{-1} mat(b) - const Mat Yt = Y.transpose(); - Mat X = luA.solve(Yt).transpose(); // (S1^{-1} Y^T)^T = Y S1^{-T} - VERIFY_IS_EQUAL(x, Vec(X.reshaped())); -} - // An ill-conditioned sparse factor: the tridiagonal Laplacian of order n has // condition number ~0.4 n^2, so I_p (x) L is judged by its backward error // ||b - K x|| <= n eps ||L||_2 ||x||, with ||L||_2 < 4, not entrywise. @@ -1522,88 +1468,72 @@ } // The decomposition family on a sparse factor (materialized densely for the -// factor decompositions): eigen-residual, SVD reconstruction, rank and -// minimum-norm least squares against the dense references. +// factor decompositions) against the dense references, one test per +// decomposition to keep each part's compile small. template <typename Scalar> -void test_kron_sparse_decompositions(Index n1, Index n2) { +void test_kron_sparse_eigen(Index n1, Index n2) { using RealScalar = typename NumTraits<Scalar>::Real; using Complex = std::complex<RealScalar>; - using Vec = Matrix<Scalar, Dynamic, 1>; using Mat = Matrix<Scalar, Dynamic, Dynamic>; using CMat = Matrix<Complex, Dynamic, Dynamic>; using CVec = Matrix<Complex, Dynamic, 1>; using Sparse = SparseMatrix<Scalar>; const Sparse S = random_sparse_dominant<Scalar>(n1); - Mat B = Mat::Random(n2, n2); - KroneckerOperator<Sparse, Mat> K(S, B); - Mat dense = reference_kron<Scalar>(Mat(S), B); - - CVec lambda = K.eigenvalues(); - CMat V = K.eigenvectors(); + const Mat B = Mat::Random(n2, n2); + const KroneckerOperator<Sparse, Mat> K(S, B); + const Mat dense = reference_kron<Scalar>(Mat(S), B); + const CVec lambda = K.eigenvalues(); + const CMat V = K.eigenvectors(); VERIFY_IS_APPROX((dense.template cast<Complex>() * V).eval(), (V * lambda.asDiagonal()).eval()); +} - Matrix<RealScalar, Dynamic, 1> sv = K.singularValues(); - Mat U = K.matrixU(), W = K.matrixV(); +template <typename Scalar> +void test_kron_sparse_svd(Index n1, Index n2) { + using RealScalar = typename NumTraits<Scalar>::Real; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using Sparse = SparseMatrix<Scalar>; + + const Sparse S = random_sparse_dominant<Scalar>(n1); + const Mat B = Mat::Random(n2, n2); + const KroneckerOperator<Sparse, Mat> K(S, B); + const Mat dense = reference_kron<Scalar>(Mat(S), B); + const Matrix<RealScalar, Dynamic, 1> sv = K.singularValues(); + const Mat U = K.matrixU(), W = K.matrixV(); VERIFY_IS_APPROX((U * sv.template cast<Scalar>().asDiagonal() * W.adjoint()).eval(), dense); +} - VERIFY_IS_EQUAL(K.rank(), dense.completeOrthogonalDecomposition().rank()); - Vec b = Vec::Random(n1 * n2); +template <typename Scalar> +void test_kron_sparse_rank(Index n1, Index n2) { + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using Sparse = SparseMatrix<Scalar>; + + const Sparse S = random_sparse_dominant<Scalar>(n1); + const KroneckerOperator<Sparse, Mat> K(S, Mat::Random(n2, n2)); + VERIFY_IS_EQUAL(K.rank(), Mat(K).completeOrthogonalDecomposition().rank()); +} + +template <typename Scalar> +void test_kron_sparse_least_squares(Index n1, Index n2) { + using Vec = Matrix<Scalar, Dynamic, 1>; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using Sparse = SparseMatrix<Scalar>; + + const Sparse S = random_sparse_dominant<Scalar>(n1); + const Mat B = Mat::Random(n2, n2); + const KroneckerOperator<Sparse, Mat> K(S, B); + const Mat dense = reference_kron<Scalar>(Mat(S), B); + const Vec b = Vec::Random(n1 * n2); VERIFY_IS_APPROX(K.leastSquaresSolve(b), dense.completeOrthogonalDecomposition().solve(b).eval()); } -// The overflow-hardening contract through sparse factors: products and solves -// with extreme magnitudes stay finite whenever the true result is -// representable, and the balanced SparseLU determinant survives partial -// products that overflow on their own. -void test_kron_sparse_extreme_scale() { +// The balanced SparseLU determinant survives partial products that overflow on +// their own. +void test_kron_sparse_determinant_scaling() { using Vec = Matrix<double, Dynamic, 1>; using Mat = Matrix<double, Dynamic, Dynamic>; using Sparse = SparseMatrix<double>; - // [1e-200] (x) [1e200] is the identity; the unprotected B X = 1e400 overflows. - Mat Sd(1, 1), B(1, 1); - Sd << 1e-200; - B << 1e200; - const Sparse S = Sd.sparseView(); - KroneckerOperator<Sparse, Mat> K(S, B); - KroneckerOperator<Mat, Sparse> K2(B, S); // the sparse factor on the applied side - Vec x(1); - x << 1e200; - Vec y = K * x; - VERIFY(y.allFinite()); - VERIFY_IS_APPROX(y[0], 1e200); - y = K2 * x; - VERIFY(y.allFinite()); - VERIFY_IS_APPROX(y[0], 1e200); - - // Same magnitudes through solve(), the sparse factor factorized in the - // normalized frame. - Vec b(1); - b << 1e200; - Vec xs = K.solve(b); - VERIFY(xs.allFinite()); - VERIFY_IS_APPROX(xs[0], 1e200); - xs = K2.solve(b); - VERIFY(xs.allFinite()); - VERIFY_IS_APPROX(xs[0], 1e200); - - // [[0, 1e-200], [1e-200, 0]] (x) [1e200] is the 2 x 2 exchange matrix: the - // scaled factorization pivots off the diagonal and every intermediate stays - // finite through product and solve. - Mat Ed(2, 2); - Ed << 0.0, 1e-200, 1e-200, 0.0; - const Sparse E = Ed.sparseView(); - KroneckerOperator<Sparse, Mat> KE(E, B); - Vec x2(2), y2(2); - x2 << 1e200, 2e200; - y2 = KE * x2; - VERIFY(y2.allFinite()); - VERIFY_IS_APPROX(y2, Vec(x2.reverse())); - Vec xe = KE.solve(x2); - VERIFY(xe.allFinite()); - VERIFY_IS_APPROX(xe, Vec(x2.reverse())); - // det([1e200])^2 = 1e400 overflows on its own, yet the determinant of // [1e200] (x) diag(1e-100, 1e-100) is 1e200 -- through the SparseLU pivots. Mat A1(1, 1); @@ -1718,30 +1648,131 @@ // Non-dyadic entries near tiny = min * 2^-(digits/2), which carry about // digits/2 bits: eliminating there would lose the other half, the exact // scale-up does not. The reference is the determinant of the stored data. - // ARMv7 NEON flushes subnormal inputs regardless of FPSCR.FZ: the packet - // exponent bound then sees a zero factor, skips the scale-up, and the - // elimination keeps only those digits/2 bits. - if (!subnormalDivisionIsExact<Real>()) return; + // Coefficient-wise arithmetic keeps the subnormal entries out of packets, + // which ARMv7 NEON flushes regardless of FPSCR.FZ. const Real tiny = std::ldexp(normalMin, -(std::numeric_limits<Real>::digits / 2)); const Real huge = std::ldexp(Real(1), std::numeric_limits<Real>::max_exponent - 1); a << Real(4.1), Real(1.3), Real(1.1), Real(5.3); - a *= Scalar(tiny) * phase; + Mat stored; sparseA.setZero(); for (Index j = 0; j < 2; ++j) - for (Index i = 0; i < 2; ++i) sparseA.insert(i, j) = a(i, j); - const Mat stored = a / tiny; // exact + for (Index i = 0; i < 2; ++i) { + a(i, j) = Scalar(numext::real(a(i, j)) * tiny) * phase; + stored(i, j) = a(i, j) / tiny; // exact + sparseA.insert(i, j) = a(i, j); + } const Scalar expectedStored = stored.determinant() * Scalar((tiny * huge) * (tiny * huge)); b << Scalar(huge); const Scalar precise = makeKroneckerOperator(sparseA, b).determinant(); VERIFY(numext::abs(precise - expectedStored) <= tolerance * numext::abs(expectedStored)); } -template <typename ProductScalar, typename Lhs, typename Rhs> +template <typename Scalar> +struct kron_subnormal_entry { + static Scalar run(const Scalar& value) { return value; } + static Scalar phaseSquared() { return Scalar(1); } +}; + +template <typename Real> +struct kron_subnormal_entry<std::complex<Real>> { + // Purely imaginary entries route the subnormal data through realView(). + static std::complex<Real> run(const Real& value) { return std::complex<Real>(Real(0), value); } + static std::complex<Real> phaseSquared() { return std::complex<Real>(Real(-1), Real(0)); } +}; + +// All-subnormal factors of every kind, built from bit patterns so that FTZ/DAZ +// cannot flush the fixture: det(a (x) b) = det(m) (denorm_min b)^2 with +// b = 2^(max_exponent - 1), a normal reference, plainly and under flush-to-zero. +template <typename Scalar> +void test_kron_determinant_flushed_subnormal() { + using Real = typename NumTraits<Scalar>::Real; + using Bits = typename internal::binary_floating_point_traits<Real>::Bits; + using Entry = kron_subnormal_entry<Scalar>; + using Mat = Matrix<Scalar, 2, 2>; + using Sparse = SparseMatrix<Scalar>; + // a = m * denorm_min with m = [4113 1301; 1103 5309], det m = 20400914: digits/2 bits per entry. + const Bits significands[2][2] = {{Bits(4113), Bits(1301)}, {Bits(1103), Bits(5309)}}; + const double detSignificands = 4113.0 * 5309.0 - 1301.0 * 1103.0; + const double detDiagonalSignificands = 4113.0 * 5309.0; + // denorm_min * b = 2^(min_exponent - digits) * 2^(max_exponent - 1), a normal power of two. + const Real scaledUnit = + numext::ldexp(Real(1), std::numeric_limits<Real>::min_exponent - std::numeric_limits<Real>::digits + + std::numeric_limits<Real>::max_exponent - 1); + const Scalar expected = Scalar(Real(detSignificands) * scaledUnit * scaledUnit) * Entry::phaseSquared(); + const Scalar expectedDiagonal = + Scalar(Real(detDiagonalSignificands) * scaledUnit * scaledUnit) * Entry::phaseSquared(); + // The LU rounds each pivot a few times and the balanced accumulation once per factor. + const Real tolerance = Real(64) * NumTraits<Real>::epsilon(); + + Mat a; + for (Index i = 0; i < 2; ++i) + for (Index j = 0; j < 2; ++j) a(i, j) = Entry::run(numext::bit_cast<Real>(significands[i][j])); + Sparse sparseA(2, 2); + for (Index j = 0; j < 2; ++j) + for (Index i = 0; i < 2; ++i) sparseA.insert(i, j) = a(i, j); + const DiagonalMatrix<Scalar, 2> diagonalA(a(0, 0), a(1, 1)); + Matrix<Scalar, 1, 1> b; + b << Scalar(numext::ldexp(Real(1), std::numeric_limits<Real>::max_exponent - 1)); + + const auto check = [&]() { + const auto close = [&](const Scalar& actual, const Scalar& reference) { + VERIFY((numext::isfinite)(actual)); + VERIFY(numext::abs(actual - reference) <= tolerance * numext::abs(reference)); + }; + close(makeKroneckerOperator(a, b).determinant(), expected); + close(makeKroneckerOperator(b, a).determinant(), expected); + close(makeKroneckerOperator(sparseA, b).determinant(), expected); + close(makeKroneckerOperator(b, sparseA).determinant(), expected); + close(makeKroneckerOperator(diagonalA, b).determinant(), expectedDiagonal); + close(makeKroneckerOperator(b, diagonalA).determinant(), expectedDiagonal); + }; + forEachFlushToZeroMode([&](FlushToZeroMode) { check(); }); +} + +// A factor with a normal and a subnormal pivot 40 octaves apart: the packet +// scale-up would flush the small pivot under flush-to-zero, and the exact one +// keeps det(A (x) b) = 2^-100 2^-140 (2^120)^2 = 1 for every factor kind. +template <typename Scalar> +void test_kron_determinant_flushed_pivot() { + using Real = typename NumTraits<Scalar>::Real; + using Entry = kron_subnormal_entry<Scalar>; + using Mat = Matrix<Scalar, 2, 2>; + using Sparse = SparseMatrix<Scalar>; + constexpr int minExponent = std::numeric_limits<Real>::min_exponent; + // 2^(min_exponent + 25) and 2^(min_exponent - 15): 40 octaves apart, the second one subnormal. + const Real large = numext::ldexp(Real(1), minExponent + 25); + const Real small = numext::bit_cast<Real>(typename internal::binary_floating_point_traits<Real>::Bits(1) + << (std::numeric_limits<Real>::digits - 15)); + const Real b = numext::ldexp(Real(1), -minExponent - 5); // large * small * b^2 == 1 + Mat a = Mat::Zero(); + a(0, 0) = Entry::run(large); + a(1, 1) = Entry::run(small); + Sparse sparseA(2, 2); + sparseA.insert(0, 0) = a(0, 0); + sparseA.insert(1, 1) = a(1, 1); + const DiagonalMatrix<Scalar, 2> diagonalA(a(0, 0), a(1, 1)); + Matrix<Scalar, 1, 1> bMat; + bMat << Scalar(b); + const Scalar expected = Entry::phaseSquared(); + const Real tolerance = Real(16) * NumTraits<Real>::epsilon(); + const auto check = [&]() { + const auto close = [&](const Scalar& actual) { + VERIFY(numext::abs(actual - expected) <= tolerance * numext::abs(expected)); + }; + close(makeKroneckerOperator(a, bMat).determinant()); + close(makeKroneckerOperator(sparseA, bMat).determinant()); + close(makeKroneckerOperator(diagonalA, bMat).determinant()); + }; + forEachFlushToZeroMode([&](FlushToZeroMode) { check(); }); +} + +// Materialization is entrywise, so it stays well-defined for non-finite +// factors: coefficient access, dense and sparse assignment must agree (products +// carry no such contract; Inf/NaN propagate as through the factor products). +template <typename Lhs, typename Rhs> void check_kron_sparse_nonfinite(const KroneckerOperator<Lhs, Rhs>& k) { using Scalar = typename Lhs::Scalar; - using Real = typename NumTraits<ProductScalar>::Real; using Mat = Matrix<Scalar, Dynamic, Dynamic>; - using ProductMatrix = Matrix<ProductScalar, Dynamic, Dynamic, RowMajor>; SparseMatrix<Scalar> sparse; sparse = k; const Mat dense = k; @@ -1750,23 +1781,6 @@ for (Index i = 0; i < k.rows(); ++i) coefficients(i, j) = k.coeff(i, j); VERIFY_IS_CWISE_EQUAL(coefficients.realView(), dense.realView()); VERIFY_IS_CWISE_EQUAL(Mat(sparse).realView(), dense.realView()); - - ProductMatrix x = ProductMatrix::Ones(k.cols(), 3); - x(0, 1) = ProductScalar(NumTraits<Real>::quiet_NaN()); - x(k.cols() - 1, 2) = ProductScalar(NumTraits<Real>::infinity()); - // Vector products keep the reference independent of complex packet Inf/NaN handling. - ProductMatrix expected(k.rows(), x.cols()); - for (Index j = 0; j < x.cols(); ++j) expected.col(j) = sparse * x.col(j); - ProductMatrix actual = k * x; - VERIFY_IS_CWISE_EQUAL(actual.realView(), expected.realView()); - actual.setConstant(ProductScalar(3)); - actual.noalias() += k * x; - const ProductMatrix added = (expected.array() + ProductScalar(3)).matrix(); - VERIFY_IS_CWISE_EQUAL(actual.realView(), added.realView()); - actual.setConstant(ProductScalar(3)); - actual.noalias() -= k * x; - const ProductMatrix subtracted = (ProductScalar(3) - expected.array()).matrix(); - VERIFY_IS_CWISE_EQUAL(actual.realView(), subtracted.realView()); } template <typename Scalar, int Options> @@ -1783,36 +1797,293 @@ pattern.insert(0, 2) = Scalar(2); // Row 1 and column 1 are structurally empty. Mat finite = Mat::Ones(2, 2); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(finite, pattern)); - check_kron_sparse_nonfinite<std::complex<Real>>(makeKroneckerOperator(finite, pattern)); + check_kron_sparse_nonfinite(makeKroneckerOperator(finite, pattern)); for (Real special : {NumTraits<Real>::infinity(), NumTraits<Real>::quiet_NaN()}) { const Mat single = Mat::Constant(1, 1, Scalar(special)); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(single, empty)); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(empty, single)); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(single, storedZero)); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(storedZero, single)); + check_kron_sparse_nonfinite(makeKroneckerOperator(single, empty)); + check_kron_sparse_nonfinite(makeKroneckerOperator(empty, single)); + check_kron_sparse_nonfinite(makeKroneckerOperator(single, storedZero)); + check_kron_sparse_nonfinite(makeKroneckerOperator(storedZero, single)); Mat nonfinite = finite; nonfinite(0, 0) = Scalar(special); const Sparse sparseNonfinite = nonfinite.sparseView(); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(nonfinite, pattern)); - check_kron_sparse_nonfinite<std::complex<Real>>(makeKroneckerOperator(nonfinite, pattern)); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(pattern, nonfinite)); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(sparseNonfinite, pattern)); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(pattern, sparseNonfinite)); + check_kron_sparse_nonfinite(makeKroneckerOperator(nonfinite, pattern)); + check_kron_sparse_nonfinite(makeKroneckerOperator(pattern, nonfinite)); + check_kron_sparse_nonfinite(makeKroneckerOperator(sparseNonfinite, pattern)); + check_kron_sparse_nonfinite(makeKroneckerOperator(pattern, sparseNonfinite)); const DiagonalMatrix<Scalar, 2> diagonal(nonfinite.diagonal()); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(diagonal, pattern)); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(pattern, diagonal)); - // Without a sparse factor the product stays on the GEMM path, which must - // agree with the materialization as well. Real scalars only: the scalar - // reference does not fix the Inf/NaN components of a complex packet product. - if (!NumTraits<Scalar>::IsComplex) { - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(diagonal, nonfinite)); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(nonfinite, diagonal)); - check_kron_sparse_nonfinite<Scalar>(makeKroneckerOperator(diagonal, diagonal)); - } + check_kron_sparse_nonfinite(makeKroneckerOperator(diagonal, pattern)); + check_kron_sparse_nonfinite(makeKroneckerOperator(pattern, diagonal)); } } +// Checks every product form against the dense reference: one right-hand side +// (reshaped in place), several (stacked), and the accumulating forms. +template <typename Op, typename Mat> +void check_kron_products(const Op& K, const Mat& ref) { + using Scalar = typename Mat::Scalar; + using Complex = std::complex<typename NumTraits<Scalar>::Real>; + using Vec = Matrix<Scalar, Dynamic, 1>; + using CVec = Matrix<Complex, Dynamic, 1>; + const Vec x = Vec::Random(ref.cols()); + VERIFY_IS_APPROX((K * x).eval(), (ref * x).eval()); + const Mat X = Mat::Random(ref.cols(), 3); + VERIFY_IS_APPROX((K * X).eval(), (ref * X).eval()); + Vec y = Vec::Random(ref.rows()); + const Vec y0 = y; + y.noalias() += K * x; + VERIFY_IS_APPROX(y, (y0 + ref * x).eval()); + y = y0; + y.noalias() -= K * x; + VERIFY_IS_APPROX(y, (y0 - ref * x).eval()); + // A complex right-hand side promotes the workspaces. + const CVec xc = CVec::Random(ref.cols()); + VERIFY_IS_APPROX((K * xc).eval(), (ref.template cast<Complex>() * xc).eval()); +} + +// Identity() factors are stored as their dimensions and skipped in products, +// square or rectangular, on either side. +template <typename Scalar> +void test_kron_identity_factor(Index p, Index m2, Index n2) { + using RealScalar = typename NumTraits<Scalar>::Real; + using Vec = Matrix<Scalar, Dynamic, 1>; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using Sparse = SparseMatrix<Scalar>; + using RowSparse = SparseMatrix<Scalar, RowMajor>; + using Id = internal::kron_identity_factor<Scalar, Dynamic, Dynamic>; + + const Mat A = Mat::Random(m2, n2); + const Sparse S = random_sparse<Scalar>(m2, n2); + const Mat Ip = Mat::Identity(p, p), Sd = S; + auto KI = makeKroneckerOperator(Mat::Identity(p, p), A); // I_p (x) A + auto IK = makeKroneckerOperator(S, Mat::Identity(p, p)); // S (x) I_p + STATIC_CHECK((std::is_same<decltype(KI), KroneckerOperator<Id, Mat>>::value)); + STATIC_CHECK((std::is_same<decltype(IK), KroneckerOperator<Sparse, Id>>::value)); + const Mat refI = reference_kron<Scalar>(Ip, A), refK = reference_kron<Scalar>(Sd, Ip); + + check_kron_products(KI, refI); + check_kron_products(IK, refK); + const Vec x = Vec::Random(p * n2); + VERIFY_IS_APPROX((KI * x).eval(), (makeKroneckerOperator(Vec::Ones(p).asDiagonal(), A) * x).eval()); + + // Unlike the Identity() expression, the stored identity is assignable, and so + // is an operator holding one. + auto KA = makeKroneckerOperator(Mat::Identity(p + 1, p + 1), A); + KA = KI; + VERIFY_IS_EQUAL(Mat(KA), refI); + + // Materialization copies the other factor's entries, with p copies of each. + VERIFY_IS_EQUAL(Mat(KI), refI); + VERIFY_IS_EQUAL(Mat(IK), refK); + Sparse MI; + MI = KI; + VERIFY_IS_EQUAL(MI.nonZeros(), p * m2 * n2); + VERIFY_IS_EQUAL(Mat(MI), refI); + RowSparse MK; + MK = IK; + VERIFY_IS_EQUAL(MK.nonZeros(), p * S.nonZeros()); + VERIFY_IS_EQUAL(Mat(MK), refK); + for (Index t = 0; t < 5; ++t) { + const Index i = internal::random<Index>(0, KI.rows() - 1), j = internal::random<Index>(0, KI.cols() - 1); + VERIFY_IS_EQUAL(KI.coeff(i, j), refI(i, j)); + } + + // The transposition family keeps the identity. + STATIC_CHECK((std::is_same<decltype(KI.transpose()), KroneckerOperator<Id, Mat>>::value)); + VERIFY_IS_EQUAL(Mat(KI.transpose()), Mat(refI.transpose())); + VERIFY_IS_EQUAL(Mat(IK.adjoint()), Mat(refK.adjoint())); + VERIFY_IS_EQUAL(Mat(IK.conjugate()), Mat(refK.conjugate())); + + // Rectangular identities: I_{p x (p+2)} (x) A and A (x) I_{(p+1) x p}. + const Mat Iw = Mat::Identity(p, p + 2), It = Mat::Identity(p + 1, p); + auto KW = makeKroneckerOperator(Mat::Identity(p, p + 2), A); + auto KT = makeKroneckerOperator(A, Mat::Identity(p + 1, p)); + const Mat refW = reference_kron<Scalar>(Iw, A), refT = reference_kron<Scalar>(A, It); + check_kron_products(KW, refW); + check_kron_products(KT, refT); + check_kron_products(KW.adjoint(), Mat(refW.adjoint())); + check_kron_products(KT.transpose(), Mat(refT.transpose())); + Sparse MT; + MT = KT; + VERIFY_IS_EQUAL(MT.nonZeros(), p * m2 * n2); + VERIFY_IS_EQUAL(Mat(MT), refT); + VERIFY_IS_EQUAL(Mat(KW), refW); + + // Square factors: the identity solves, inverts and has determinant one. + const Mat B = Mat::Random(m2, m2) + RealScalar(2) * Mat::Identity(m2, m2); + auto KB = makeKroneckerOperator(Mat::Identity(p, p), B); + const Mat refB = reference_kron<Scalar>(Ip, B); + const Vec b = Vec::Random(p * m2); + VERIFY_IS_APPROX((refB * KB.solve(b)).eval(), b); + const Mat Bb = Mat::Random(p * m2, 3); + VERIFY_IS_APPROX((reference_kron<Scalar>(B, Ip) * makeKroneckerOperator(B, Mat::Identity(p, p)).solve(Bb)).eval(), + Bb); + VERIFY_IS_APPROX((Mat(KB.inverse()) * refB).eval(), Mat(Mat::Identity(p * m2, p * m2))); + VERIFY_IS_APPROX(KB.determinant(), refB.determinant()); + STATIC_CHECK( + (std::is_same<decltype(KB.inverse()), KroneckerOperator<Id, Matrix<Scalar, Dynamic, Dynamic, ColMajor>>>::value)); + VERIFY_IS_EQUAL(makeKroneckerOperator(Mat::Identity(p, p), Mat::Identity(m2, m2)).determinant(), Scalar(1)); +} + +// Nested factors: A (x) (B (x) C) and (A (x) B) (x) C against the dense +// reference, with dense, sparse and identity leaves. Moderate diagonal boost: +// invertible, with determinants tame enough for a relative comparison. The +// operation families are separate tests to keep each part's compile small. +template <typename Scalar> +struct NestedKronecker { + using RealScalar = typename NumTraits<Scalar>::Real; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using Sparse = SparseMatrix<Scalar>; + using RightNested = KroneckerOperator<Mat, KroneckerOperator<Sparse, Mat>>; + using LeftNested = KroneckerOperator<KroneckerOperator<Mat, Sparse>, Mat>; + + NestedKronecker(Index n1, Index n2, Index n3) + : A(Mat::Random(n1, n1) + RealScalar(2) * Mat::Identity(n1, n1)), + Bs(random_sparse_dominant<Scalar>(n2)), + C(Mat::Random(n3, n3) + RealScalar(2) * Mat::Identity(n3, n3)), + Bd(Bs), + ref(reference_kron<Scalar>(A, reference_kron<Scalar>(Bd, C))), + KR(makeKroneckerOperator(A, Bs, C)), + KL(makeKroneckerOperator(A, Bs), C) {} + + Mat A; + Sparse Bs; + Mat C, Bd, ref; + RightNested KR; + LeftNested KL; +}; + +template <typename Scalar> +void test_kron_nested_products(Index n1, Index n2, Index n3) { + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using Sparse = SparseMatrix<Scalar>; + using RowSparse = SparseMatrix<Scalar, RowMajor>; + using Id = internal::kron_identity_factor<Scalar, Dynamic, Dynamic>; + const NestedKronecker<Scalar> f(n1, n2, n3); + const Mat& ref = f.ref; + + STATIC_CHECK((std::is_same<decltype(makeKroneckerOperator(f.A, f.Bs, f.C)), + typename NestedKronecker<Scalar>::RightNested>::value)); + VERIFY_IS_EQUAL(f.KR.rows(), n1 * n2 * n3); + VERIFY_IS_EQUAL(f.KL.cols(), n1 * n2 * n3); + check_kron_products(f.KR, ref); + check_kron_products(f.KL, ref); + VERIFY_IS_APPROX(Mat(f.KR), ref); + VERIFY_IS_APPROX(Mat(f.KL), ref); + Mat acc = Mat::Random(ref.rows(), ref.cols()); + const Mat acc0 = acc; + acc += f.KL; + acc -= f.KR; + VERIFY_IS_APPROX(acc, acc0); + for (Index t = 0; t < 5; ++t) { + const Index i = internal::random<Index>(0, ref.rows() - 1), j = internal::random<Index>(0, ref.cols() - 1); + VERIFY_IS_APPROX(f.KR.coeff(i, j), ref(i, j)); + VERIFY_IS_APPROX(f.KL.coeff(i, j), ref(i, j)); + } + + // Sparse materialization stores the products of the leaves' stored entries. + const Index nnz = n1 * n1 * f.Bs.nonZeros() * n3 * n3; + Sparse M; + M = f.KR; + VERIFY_IS_EQUAL(M.nonZeros(), nnz); + VERIFY_IS_APPROX(Mat(M), ref); + RowSparse MR; + MR = f.KL; + VERIFY_IS_EQUAL(MR.nonZeros(), nnz); + VERIFY_IS_APPROX(Mat(MR), ref); + + // The transposition family recurses into the nested factor. + check_kron_products(f.KR.transpose(), Mat(ref.transpose())); + check_kron_products(f.KL.adjoint(), Mat(ref.adjoint())); + check_kron_products(f.KR.conjugate(), Mat(ref.conjugate())); + + // The identity sandwich I_p (x) S (x) I_q of a 3-D finite-difference term. + const Index p = n1, q = n3; + auto K3 = makeKroneckerOperator(Mat::Identity(p, p), f.Bs, Mat::Identity(q, q)); + STATIC_CHECK((std::is_same<decltype(K3), KroneckerOperator<Id, KroneckerOperator<Sparse, Id>>>::value)); + const Mat ref3 = reference_kron<Scalar>(Mat::Identity(p, p), reference_kron<Scalar>(f.Bd, Mat::Identity(q, q))); + check_kron_products(K3, ref3); + Sparse M3; + M3 = K3; + VERIFY_IS_EQUAL(M3.nonZeros(), p * q * f.Bs.nonZeros()); + VERIFY_IS_EQUAL(Mat(M3), ref3); +} + +// A left-nested factor applies from the right: with a square identity on +// either side of it, and with a nested factor of its own. +template <typename Scalar> +void test_kron_left_nested_products(Index n1, Index n2, Index n3) { + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + const NestedKronecker<Scalar> f(n1, n2, n3); + const Index p = n1, q = n3; + const Mat Ip = Mat::Identity(p, p), Iq = Mat::Identity(q, q); + check_kron_products(makeKroneckerOperator(makeKroneckerOperator(Mat::Identity(p, p), f.Bs), Mat::Identity(q, q)), + reference_kron<Scalar>(reference_kron<Scalar>(Ip, f.Bd), Iq)); + check_kron_products(makeKroneckerOperator(makeKroneckerOperator(f.Bs, Mat::Identity(q, q)), f.C), + reference_kron<Scalar>(reference_kron<Scalar>(f.Bd, Iq), f.C)); + check_kron_products(makeKroneckerOperator(f.KR, f.C), reference_kron<Scalar>(f.ref, f.C)); +} + +// Solves factorize each leaf once; the inverse and determinant recurse. +template <typename Scalar> +void test_kron_nested_solve(Index n1, Index n2, Index n3) { + using Vec = Matrix<Scalar, Dynamic, 1>; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + const NestedKronecker<Scalar> f(n1, n2, n3); + const Mat& ref = f.ref; + const Vec b = Vec::Random(ref.rows()); + VERIFY_IS_APPROX((ref * f.KR.solve(b)).eval(), b); + const Mat Bm = Mat::Random(ref.rows(), 3); + VERIFY_IS_APPROX((ref * f.KL.solve(Bm)).eval(), Bm); + VERIFY_IS_APPROX((Mat(f.KR.inverse()) * ref).eval(), Mat(Mat::Identity(ref.rows(), ref.cols()))); + VERIFY_IS_APPROX(f.KR.determinant(), ref.determinant()); + VERIFY_IS_APPROX(f.KL.determinant(), ref.determinant()); + + const Index p = n1, q = n3; + const auto K3 = makeKroneckerOperator(Mat::Identity(p, p), f.Bs, Mat::Identity(q, q)); + const Mat ref3 = reference_kron<Scalar>(Mat::Identity(p, p), reference_kron<Scalar>(f.Bd, Mat::Identity(q, q))); + const Vec b3 = Vec::Random(ref3.rows()); + VERIFY_IS_APPROX((ref3 * K3.solve(b3)).eval(), b3); +} + +// The eigendecomposition and the SVD recurse into the nested factor. +template <typename Scalar> +void test_kron_nested_eigen(Index n1, Index n2, Index n3) { + using Complex = std::complex<typename NumTraits<Scalar>::Real>; + using CMat = Matrix<Complex, Dynamic, Dynamic>; + using CVec = Matrix<Complex, Dynamic, 1>; + using ColMajorCMat = Matrix<Complex, Dynamic, Dynamic, ColMajor>; + const NestedKronecker<Scalar> f(n1, n2, n3); + const CVec lambda = f.KL.eigenvalues(); + const CMat V = f.KL.eigenvectors(); + VERIFY_IS_APPROX((f.ref.template cast<Complex>() * V).eval(), (V * lambda.asDiagonal()).eval()); + STATIC_CHECK((std::is_same<decltype(f.KR.eigenvectors()), + KroneckerOperator<ColMajorCMat, KroneckerOperator<ColMajorCMat, ColMajorCMat>>>::value)); + const CVec mu = f.KR.eigenvalues(); + const CMat W = f.KR.eigenvectors(); + VERIFY_IS_APPROX((f.ref.template cast<Complex>() * W).eval(), (W * mu.asDiagonal()).eval()); +} + +template <typename Scalar> +void test_kron_nested_svd(Index n1, Index n2, Index n3) { + using RealScalar = typename NumTraits<Scalar>::Real; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using RealVec = Matrix<RealScalar, Dynamic, 1>; + const NestedKronecker<Scalar> f(n1, n2, n3); + const Mat& ref = f.ref; + const RealVec s = f.KR.singularValues(); + RealVec sv = s; + std::sort(sv.data(), sv.data() + sv.size(), std::greater<RealScalar>()); + VERIFY_IS_APPROX(sv, RealVec(BDCSVD<Mat>(ref).singularValues())); + VERIFY_IS_EQUAL(f.KR.rank(), ref.rows()); + // U and V nest like the operator and pair with the unsorted singular values. + const Mat U = f.KR.matrixU(), V = f.KR.matrixV(); + VERIFY_IS_APPROX((U * s.asDiagonal() * V.adjoint()).eval(), ref); + VERIFY_IS_APPROX((U.adjoint() * U).eval(), Mat(Mat::Identity(ref.cols(), ref.cols()))); + const Mat UL = f.KL.matrixU(), VL = f.KL.matrixV(); + VERIFY_IS_APPROX((UL * f.KL.singularValues().asDiagonal() * VL.adjoint()).eval(), ref); +} + EIGEN_DECLARE_TEST(structured_kronecker) { for (int i = 0; i < g_repeat; ++i) { // Products, dense assignment, coefficient access across factor shapes. @@ -1833,53 +2104,56 @@ CALL_SUBTEST_3((test_kron_solve<double>(1, 1))); CALL_SUBTEST_3((test_kron_solve<double>(4, 7))); CALL_SUBTEST_3((test_kron_solve<double>(9, 5))); - CALL_SUBTEST_3((test_kron_solve<float>(5, 4))); - CALL_SUBTEST_3((test_kron_solve<std::complex<double>>(6, 4))); - CALL_SUBTEST_3(test_kron_solve_normalized()); - CALL_SUBTEST_3((test_kron_solve_bit_identity<double>(4, 3))); - CALL_SUBTEST_3((test_kron_solve_bit_identity<std::complex<double>>(3, 4))); + CALL_SUBTEST_15((test_kron_solve<float>(5, 4))); + CALL_SUBTEST_15((test_kron_solve<std::complex<double>>(6, 4))); // Minimum-norm least squares through factor pseudo-inverses. CALL_SUBTEST_3((test_kron_least_squares<double>(8, 5, 7, 4))); // tall x tall CALL_SUBTEST_3((test_kron_least_squares<double>(8, 5, 3, 6))); // tall x wide - CALL_SUBTEST_3((test_kron_least_squares<std::complex<double>>(6, 4, 5, 3))); + CALL_SUBTEST_26((test_kron_least_squares<std::complex<double>>(6, 4, 5, 3))); CALL_SUBTEST_3((test_kron_least_squares_rank_deficient<double>(6, 4, 5, 3))); - CALL_SUBTEST_3((test_kron_least_squares_rank_deficient<std::complex<double>>(5, 3, 4, 4))); + CALL_SUBTEST_27((test_kron_least_squares_rank_deficient<std::complex<double>>(5, 3, 4, 4))); + CALL_SUBTEST_3((test_kron_least_squares<float>(6, 4, 5, 3))); + CALL_SUBTEST_3((test_kron_least_squares_scaling<double>(6, 4, 3, 5))); + CALL_SUBTEST_28((test_kron_least_squares_scaling<std::complex<float>>(4, 5, 6, 3))); // Eigendecomposition, SVD, inverse, determinant. CALL_SUBTEST_4((test_kron_eigen<double>(4, 5))); CALL_SUBTEST_4((test_kron_eigen<double>(1, 6))); - CALL_SUBTEST_4((test_kron_eigen<std::complex<double>>(3, 4))); + CALL_SUBTEST_16((test_kron_eigen<std::complex<double>>(3, 4))); CALL_SUBTEST_4((test_kron_svd<double>(4, 4, 5, 5))); CALL_SUBTEST_4((test_kron_svd<double>(6, 4, 3, 5))); // rectangular: structural zeros - CALL_SUBTEST_4((test_kron_svd<std::complex<double>>(4, 3, 4, 4))); - CALL_SUBTEST_4((test_kron_svd<float>(4, 4, 4, 4))); + CALL_SUBTEST_20((test_kron_svd<std::complex<double>>(4, 3, 4, 4))); + CALL_SUBTEST_29((test_kron_svd<float>(4, 4, 4, 4))); CALL_SUBTEST_4((test_kron_inverse_determinant<double>(4, 5))); - CALL_SUBTEST_4((test_kron_inverse_determinant<std::complex<double>>(3, 4))); + CALL_SUBTEST_30((test_kron_inverse_determinant<std::complex<double>>(3, 4))); // Matrix-free iterative solve and fixed-size factors. CALL_SUBTEST_5((test_kron_matrix_free_cg<double>(6, 7))); CALL_SUBTEST_5((test_kron_fixed<double, 3, 4, 2, 5>())); - CALL_SUBTEST_5((test_kron_fixed<std::complex<float>, 2, 3, 3, 2>())); + CALL_SUBTEST_17((test_kron_fixed<std::complex<float>, 2, 3, 3, 2>())); CALL_SUBTEST_5((test_kron_fixed_solve<double, 2, 3>())); - CALL_SUBTEST_5((test_kron_fixed_solve<std::complex<double>, 3, 2>())); + CALL_SUBTEST_17((test_kron_fixed_solve<std::complex<double>, 3, 2>())); // Lifetime and aliasing of the product expression, mixed-scalar promotion. CALL_SUBTEST_6((test_kron_aliased_product<double>(3, 4))); - CALL_SUBTEST_6((test_kron_aliased_product<std::complex<double>>(4, 3))); + CALL_SUBTEST_18((test_kron_aliased_product<std::complex<double>>(4, 3))); CALL_SUBTEST_6((test_kron_aliased_resize<double>(4, 3, 2, 5))); - CALL_SUBTEST_6((test_kron_aliased_resize<std::complex<double>>(2, 4, 5, 3))); + CALL_SUBTEST_18((test_kron_aliased_resize<std::complex<double>>(2, 4, 5, 3))); CALL_SUBTEST_6((test_kron_delayed_product<double>(4, 3, 2, 5))); - CALL_SUBTEST_6((test_kron_delayed_product<std::complex<float>>(3, 3, 4, 2))); + CALL_SUBTEST_18((test_kron_delayed_product<std::complex<float>>(3, 3, 4, 2))); CALL_SUBTEST_6((test_kron_mixed_scalar<double>(4, 3, 3, 5))); - CALL_SUBTEST_6((test_kron_mixed_scalar<float>(3, 4, 2, 3))); + CALL_SUBTEST_18((test_kron_mixed_scalar<float>(3, 4, 2, 3))); + CALL_SUBTEST_19((test_kron_chunked_products<double>())); + CALL_SUBTEST_31((test_kron_chunked_solves<double>())); + CALL_SUBTEST_32((test_kron_chunked_products<std::complex<double>>())); + CALL_SUBTEST_33((test_kron_chunked_solves<std::complex<double>>())); + CALL_SUBTEST_31((test_kron_chunked_mixed_scalar<double>())); - // Numerical boundaries: product-level rank truncation, balanced determinant, - // overflow-safe product scaling and ratio-space rank thresholds. + // Numerical boundaries: product-level rank truncation, balanced determinant + // and ratio-space rank thresholds. CALL_SUBTEST_7(test_kron_product_level_rank()); CALL_SUBTEST_7(test_kron_determinant_scaling()); - CALL_SUBTEST_7(test_kron_extreme_scale_product()); - CALL_SUBTEST_7(test_kron_extreme_scale_complex()); CALL_SUBTEST_7(test_kron_rank_ratio_threshold()); CALL_SUBTEST_7(test_kron_rank_min_normal_clamp()); @@ -1898,47 +2172,57 @@ CALL_SUBTEST_8((test_kron_diag_fixed<std::complex<float>, 2, 3, 3>())); // Diagonal factors through the solvers, inverse/determinant, the - // decomposition family, extreme scales and matrix-free CG. + // decomposition family and matrix-free CG. CALL_SUBTEST_9((test_kron_diag_solve<double>(4, 7))); - CALL_SUBTEST_9((test_kron_diag_solve<float>(5, 4))); - CALL_SUBTEST_9((test_kron_diag_solve<std::complex<double>>(6, 4))); + CALL_SUBTEST_21((test_kron_diag_solve<float>(5, 4))); + CALL_SUBTEST_21((test_kron_diag_solve<std::complex<double>>(6, 4))); CALL_SUBTEST_9((test_kron_diag_inverse_determinant<double>(4, 5))); - CALL_SUBTEST_9((test_kron_diag_inverse_determinant<std::complex<double>>(3, 4))); - CALL_SUBTEST_9((test_kron_diag_decompositions<double>(4, 5))); - CALL_SUBTEST_9((test_kron_diag_decompositions<std::complex<double>>(3, 4))); - CALL_SUBTEST_9(test_kron_diag_extreme_scale()); + CALL_SUBTEST_30((test_kron_diag_inverse_determinant<std::complex<double>>(3, 4))); + CALL_SUBTEST_9((test_kron_diag_eigen<double>(4, 5))); + CALL_SUBTEST_9((test_kron_diag_svd<double>(4, 5))); + CALL_SUBTEST_9((test_kron_diag_rank<double>(4, 5))); + CALL_SUBTEST_9((test_kron_diag_least_squares<double>(4, 5))); + CALL_SUBTEST_34((test_kron_diag_eigen<std::complex<double>>(3, 4))); + CALL_SUBTEST_35((test_kron_diag_svd<std::complex<double>>(3, 4))); + CALL_SUBTEST_36((test_kron_diag_rank<std::complex<double>>(3, 4))); + CALL_SUBTEST_37((test_kron_diag_least_squares<std::complex<double>>(3, 4))); + CALL_SUBTEST_9(test_kron_diag_determinant_scaling()); CALL_SUBTEST_9(test_kron_diag_matrix_free_cg(6, 7)); // Sparse factors: products, materialization, the identity-Kronecker // operators, the transposition family, mixed-scalar promotion. CALL_SUBTEST_10((test_kron_sparse_product<double>(4, 5, 3, 4))); - CALL_SUBTEST_10((test_kron_sparse_product<float>(3, 4, 4, 2))); - CALL_SUBTEST_10((test_kron_sparse_product<std::complex<double>>(3, 4, 2, 3))); + CALL_SUBTEST_38((test_kron_sparse_product<float>(3, 4, 4, 2))); + CALL_SUBTEST_39((test_kron_sparse_product<std::complex<double>>(3, 4, 2, 3))); CALL_SUBTEST_10((test_kron_sparse_product<double>(1, 1, 1, 1))); CALL_SUBTEST_10((test_kron_sparse_materialize<double>(4, 5, 3, 4))); - CALL_SUBTEST_10((test_kron_sparse_materialize<std::complex<float>>(3, 2, 4, 3))); + CALL_SUBTEST_38((test_kron_sparse_materialize<std::complex<float>>(3, 2, 4, 3))); CALL_SUBTEST_10((test_kron_sparse_identity<double>(5, 4, 3, 2))); - CALL_SUBTEST_10((test_kron_sparse_identity<std::complex<double>>(3, 3, 4, 2))); + CALL_SUBTEST_39((test_kron_sparse_identity<std::complex<double>>(3, 3, 4, 2))); CALL_SUBTEST_10((test_kron_sparse_transpose<double>(4, 5, 3, 2))); - CALL_SUBTEST_10((test_kron_sparse_transpose<std::complex<double>>(3, 4, 2, 5))); + CALL_SUBTEST_40((test_kron_sparse_transpose<std::complex<double>>(3, 4, 2, 5))); CALL_SUBTEST_10((test_kron_sparse_mixed_scalar<double>(4, 3, 5))); - CALL_SUBTEST_10((test_kron_sparse_mixed_scalar<float>(3, 4, 2))); + CALL_SUBTEST_40((test_kron_sparse_mixed_scalar<float>(3, 4, 2))); CALL_SUBTEST_10(test_kron_sparse_stored_zeros(4, 3)); // Sparse factors through the solvers, inverse/determinant, the - // decomposition family, extreme scales and matrix-free CG. + // decomposition family and matrix-free CG. CALL_SUBTEST_11((test_kron_sparse_solve<double>(4, 7))); - CALL_SUBTEST_11((test_kron_sparse_solve<float>(5, 4))); - CALL_SUBTEST_11((test_kron_sparse_solve<std::complex<double>>(6, 4))); + CALL_SUBTEST_22((test_kron_sparse_solve<float>(5, 4))); + CALL_SUBTEST_22((test_kron_sparse_solve<std::complex<double>>(6, 4))); CALL_SUBTEST_11((test_kron_sparse_inverse_determinant<double>(4, 5))); - CALL_SUBTEST_11((test_kron_sparse_inverse_determinant<std::complex<double>>(3, 4))); - CALL_SUBTEST_11((test_kron_sparse_decompositions<double>(4, 5))); - CALL_SUBTEST_11((test_kron_sparse_decompositions<std::complex<double>>(3, 4))); - CALL_SUBTEST_11((test_kron_sparse_solve_bit_identity<double>(4, 3))); - CALL_SUBTEST_11((test_kron_sparse_solve_bit_identity<std::complex<double>>(3, 4))); + CALL_SUBTEST_22((test_kron_sparse_inverse_determinant<std::complex<double>>(3, 4))); + CALL_SUBTEST_11((test_kron_sparse_eigen<double>(4, 5))); + CALL_SUBTEST_11((test_kron_sparse_svd<double>(4, 5))); + CALL_SUBTEST_11((test_kron_sparse_rank<double>(4, 5))); + CALL_SUBTEST_11((test_kron_sparse_least_squares<double>(4, 5))); + CALL_SUBTEST_23((test_kron_sparse_eigen<std::complex<double>>(3, 4))); + CALL_SUBTEST_41((test_kron_sparse_svd<std::complex<double>>(3, 4))); + CALL_SUBTEST_42((test_kron_sparse_rank<std::complex<double>>(3, 4))); + CALL_SUBTEST_43((test_kron_sparse_least_squares<std::complex<double>>(3, 4))); CALL_SUBTEST_11(test_kron_sparse_solve_laplacian(64, 4)); CALL_SUBTEST_11(test_kron_sparse_degenerate(4, 3)); - CALL_SUBTEST_11(test_kron_sparse_extreme_scale()); + CALL_SUBTEST_11(test_kron_sparse_determinant_scaling()); CALL_SUBTEST_11(test_kron_sparse_matrix_free_cg(6, 7)); CALL_SUBTEST_12((test_kron_sparse_determinant_subnormal<float, ColMajor>(1))); @@ -1950,5 +2234,34 @@ CALL_SUBTEST_13((test_kron_sparse_nonfinite<double, RowMajor>())); CALL_SUBTEST_13((test_kron_sparse_nonfinite<std::complex<float>, RowMajor>())); CALL_SUBTEST_13((test_kron_sparse_nonfinite<std::complex<double>, ColMajor>())); + + CALL_SUBTEST_14((test_kron_determinant_flushed_subnormal<float>())); + CALL_SUBTEST_14((test_kron_determinant_flushed_subnormal<double>())); + CALL_SUBTEST_14((test_kron_determinant_flushed_subnormal<std::complex<float>>())); + CALL_SUBTEST_14((test_kron_determinant_flushed_subnormal<std::complex<double>>())); + CALL_SUBTEST_14((test_kron_determinant_flushed_pivot<float>())); + CALL_SUBTEST_14((test_kron_determinant_flushed_pivot<double>())); + CALL_SUBTEST_14((test_kron_determinant_flushed_pivot<std::complex<float>>())); + CALL_SUBTEST_14((test_kron_determinant_flushed_pivot<std::complex<double>>())); + + // Identity() factors and nested Kronecker factors. + CALL_SUBTEST_24((test_kron_identity_factor<double>(3, 4, 5))); + CALL_SUBTEST_44((test_kron_identity_factor<float>(4, 3, 2))); + CALL_SUBTEST_45((test_kron_identity_factor<std::complex<double>>(2, 3, 4))); + CALL_SUBTEST_24((test_kron_identity_factor<double>(1, 1, 1))); + CALL_SUBTEST_25((test_kron_nested_products<double>(3, 4, 2))); + CALL_SUBTEST_31((test_kron_nested_solve<double>(3, 4, 2))); + CALL_SUBTEST_46((test_kron_nested_eigen<double>(3, 4, 2))); + CALL_SUBTEST_31((test_kron_nested_svd<double>(3, 4, 2))); + CALL_SUBTEST_47((test_kron_nested_products<std::complex<double>>(2, 3, 3))); + CALL_SUBTEST_48((test_kron_nested_solve<std::complex<double>>(2, 3, 3))); + CALL_SUBTEST_49((test_kron_nested_eigen<std::complex<double>>(2, 3, 3))); + CALL_SUBTEST_50((test_kron_nested_svd<std::complex<double>>(2, 3, 3))); + CALL_SUBTEST_51((test_kron_nested_products<float>(2, 2, 3))); + CALL_SUBTEST_52((test_kron_nested_solve<float>(2, 2, 3))); + CALL_SUBTEST_53((test_kron_nested_eigen<float>(2, 2, 3))); + CALL_SUBTEST_54((test_kron_nested_svd<float>(2, 2, 3))); + CALL_SUBTEST_55((test_kron_left_nested_products<double>(3, 4, 2))); + CALL_SUBTEST_56((test_kron_left_nested_products<std::complex<double>>(2, 3, 3))); } }
diff --git a/contrib/test/structured_matrices.cpp b/contrib/test/structured_matrices.cpp index 6a9deaf..313521a 100644 --- a/contrib/test/structured_matrices.cpp +++ b/contrib/test/structured_matrices.cpp
@@ -9,6 +9,7 @@ #include "main.h" #include "fp_control.h" +#include "structured_test_helpers.h" #include <contrib/Eigen/StructuredMatrices> @@ -195,68 +196,6 @@ VERIFY_IS_APPROX(y, (y0 + dense * x).eval()); } -// Reviewer regression (MR 2688): applying an exact 40x40 exchange matrix (as a -// Hankel operator, FFT tier) to a vector of huge finite entries must return the -// unchanged finite values, not NaNs: the FFT intermediates overflow unless each -// column is scaled by an exact power of two derived from the column's and the -// symbol's magnitudes. -void test_hankel_fft_overflow() { - typedef Matrix<double, Dynamic, 1> Vec; - const Index n = 40; - const double huge = (std::numeric_limits<double>::max)() / 16; - - // H(i,j) = h[i+j] with h = e_{n-1} is the exchange matrix (anti-identity). - Vec h = Vec::Zero(2 * n - 1); - h[n - 1] = 1.0; - Hankel<double> H(h.head(n), h.tail(n)); - - Vec x = Vec::Constant(n, huge); - Vec y = H * x; - VERIFY(y.allFinite()); - VERIFY_IS_APPROX((y / huge).eval(), Vec::Ones(n).eval()); // y == x.reverse() == x - - // A huge generator makes a huge symbol; its exponent must be scaled out too. - Vec hh = Vec::Zero(2 * n - 1); - hh[n - 1] = huge; - Hankel<double> Hh(hh.head(n), hh.tail(n)); - Vec z = Hh * Vec::Ones(n); - VERIFY(z.allFinite()); - VERIFY_IS_APPROX((z / huge).eval(), Vec::Ones(n).eval()); -} - -// The scaling exponents are derived from component-wise magnitudes: a finite -// complex value near the overflow threshold has a non-representable modulus, -// which would otherwise disable the scaling and turn an exactly representable -// product into NaN. -template <typename RealScalar> -void test_hankel_fft_complex_boundary(Index n) { - typedef std::complex<RealScalar> Complex; - typedef Matrix<RealScalar, Dynamic, 1> RVec; - typedef Matrix<Complex, Dynamic, 1> CVec; - const RealScalar kFftRoundTripTol = RealScalar(100) * NumTraits<RealScalar>::epsilon(); - const RealScalar big = RealScalar(0.75) * (std::numeric_limits<RealScalar>::max)(); - - // Exchange matrix with a complex generator: the product returns the reversed - // right-hand side -- the input itself, being constant -- even though |x_k| - // overflows. - CVec h = CVec::Zero(2 * n - 1); - h[n - 1] = Complex(1); - Hankel<Complex> H(h.head(n), h.tail(n)); - CVec x = CVec::Constant(n, Complex(big, big)); - CVec y = H * x; - VERIFY(y.allFinite()); - VERIFY(((y - x).cwiseAbs() / big).maxCoeff() <= kFftRoundTripTol); - - // A real exchange operator applied to the same complex right-hand side takes - // the mixed-scalar product path. - RVec hr = RVec::Zero(2 * n - 1); - hr[n - 1] = RealScalar(1); - Hankel<RealScalar> Hr(hr.head(n), hr.tail(n)); - y = Hr * x; - VERIFY(y.allFinite()); - VERIFY(((y - x).cwiseAbs() / big).maxCoeff() <= kFftRoundTripTol); -} - // transpose()/conjugate()/adjoint() return owning temporaries (and so does // makeHankel), so the product expression must nest the structured operand by // value: a delayed-evaluated expression has to outlive the temporary operator it @@ -389,44 +328,8 @@ VERIFY_IS_APPROX(z, (denseC * xr).eval()); } -// Entrywise IEEE comparison for the non-finite tests: NaNs match NaNs, -// infinities match by value (sign included), finite entries match to roundoff. -// VERIFY_IS_APPROX would reject any output containing NaN. -template <typename D1, typename D2> -bool ieee_entrywise_match(const D1& a, const D2& b) { - if (a.rows() != b.rows() || a.cols() != b.cols()) return false; - for (Index j = 0; j < a.cols(); ++j) - for (Index i = 0; i < a.rows(); ++i) { - const typename D1::Scalar x = a(i, j), y = b(i, j); - if (x == y) continue; // finite match or same-signed infinities - if ((x != x) && (y != y)) continue; // both NaN - if (!test_isApprox(x, y)) return false; // finite roundoff - } - return true; -} - -// Scalar-loop product: the mathematically transparent IEEE reference for the -// non-finite tests. Eigen's own vectorized complex kernels can smear a single -// infinity into NaN (Inf - Inf across the split real/imaginary accumulators), so -// the dense product is not a faithful entrywise reference for non-finite data. -template <typename Scalar> -Matrix<Scalar, Dynamic, 1> reference_product_ieee(const Matrix<Scalar, Dynamic, Dynamic>& A, - const Matrix<Scalar, Dynamic, 1>& x) { - Matrix<Scalar, Dynamic, 1> y(A.rows()); - for (Index i = 0; i < A.rows(); ++i) { - Scalar acc(0); - for (Index j = 0; j < A.cols(); ++j) acc += A(i, j) * x[j]; - y[i] = acc; - } - return y; -} - -// A single Inf or NaN in the data must propagate like the reference product -- -// through the dot products that touch it -- instead of being smeared into NaNs -// across the whole output by the transforms. The reviewer reproducer (MR 2688): -// the 40x40 exchange matrix, as a Hankel operator on the FFT tier, applied to a -// vector with one +Inf must return +Inf in the mirrored entry; the remaining -// entries are NaN through the IEEE 0*Inf terms of the dense dot products. +// An Inf or NaN in the data is never lost and does not leak into other columns +// (see nonfinite_covers_reference()). template <typename Scalar> void test_hankel_nonfinite_product(Index n) { typedef typename NumTraits<Scalar>::Real RealScalar; @@ -435,42 +338,30 @@ const RealScalar inf = std::numeric_limits<RealScalar>::infinity(); const RealScalar nan = std::numeric_limits<RealScalar>::quiet_NaN(); - // The exchange matrix is the Hankel operator generated by h = e_{n-1}. - Vec h = Vec::Zero(2 * n - 1); - h[n - 1] = Scalar(1); + Vec h = Vec::Random(2 * n - 1); Hankel<Scalar> H(h.head(n), h.tail(n)); Mat dense = reference_hankel<Scalar>(h, n, n); - // Inf in the right-hand side: the mirrored entry must be +Inf (for a complex - // operator its real part; the 0*Inf cross term makes the imaginary part NaN). Vec x = Vec::Random(n); x[n / 4] = Scalar(inf); - Vec y = H * x; - VERIFY(ieee_entrywise_match(y, reference_product_ieee(dense, x))); - VERIFY(numext::real(y[n - 1 - n / 4]) == inf); - - // NaN in the right-hand side. + VERIFY(nonfinite_covers_reference((H * x).eval(), reference_product_ieee(dense, x))); Vec xn = Vec::Random(n); xn[n - 1] = Scalar(nan); - VERIFY(ieee_entrywise_match((H * xn).eval(), reference_product_ieee(dense, xn))); + VERIFY(nonfinite_covers_reference((H * xn).eval(), reference_product_ieee(dense, xn))); - // Mixed multi-column right-hand side: the non-finite column falls back to the - // direct kernel individually while the finite column keeps the FFT path. Mat Xm(n, 2); Xm.col(0) = Vec::Random(n); Xm.col(1) = x; Mat Ym = H * Xm; VERIFY_IS_APPROX(Ym.col(0).eval(), (dense * Xm.col(0)).eval()); - VERIFY(ieee_entrywise_match(Ym.col(1).eval(), reference_product_ieee(dense, Vec(Xm.col(1))))); + VERIFY(nonfinite_covers_reference(Ym.col(1).eval(), reference_product_ieee(dense, Vec(Xm.col(1))))); - // Inf in the generating sequence: the operator itself is non-finite, whatever - // the right-hand side. Vec h2 = Vec::Random(2 * n - 1); h2[1] = Scalar(-inf); Hankel<Scalar> H2(h2.head(n), h2.tail(n)); Mat dense2 = reference_hankel<Scalar>(h2, n, n); Vec x2 = Vec::Random(n); - VERIFY(ieee_entrywise_match((H2 * x2).eval(), reference_product_ieee(dense2, x2))); + VERIFY(nonfinite_covers_reference((H2 * x2).eval(), reference_product_ieee(dense2, x2))); } // The transposed / adjoint / conjugated Hankel operators agree with the dense @@ -589,6 +480,7 @@ // calibrated with a ~100x margin against measured values. The look-ahead Levinson // solver is weakly stable: the forward error grows like eps * cond, but the // residual stays small far beyond the point where the solution itself is lost. +template <typename = void> void test_hankel_hilbert() { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<double, Dynamic, Dynamic> Mat; @@ -951,78 +843,8 @@ VERIFY_IS_APPROX(y, (dense * x).eval()); } -// The FFT product path accumulates up to p addends per transform, so a finite -// right-hand side near the overflow threshold (or a huge generator) used to -// overflow inside the transforms and return NaN for a representable result. The -// power-of-two scaled path must stay finite and accurate. -template <typename RealScalar> -void test_structured_fft_overflow(Index n) { - typedef Matrix<RealScalar, Dynamic, 1> Vec; - // FFT-transform roundoff bound for the identity round trip. - const RealScalar kFftRoundTripTol = RealScalar(100) * NumTraits<RealScalar>::epsilon(); - const RealScalar huge = (std::numeric_limits<RealScalar>::max)() / RealScalar(16); - - // Identity circulant applied to a huge finite vector returns it unchanged. - Vec c = Vec::Zero(n); - c[0] = RealScalar(1); - Circulant<RealScalar> C(c); - Vec x = Vec::Constant(n, huge); - Vec y = C * x; - VERIFY(y.allFinite()); - VERIFY(((y - x).cwiseAbs() / huge).maxCoeff() <= kFftRoundTripTol); - - // A huge generator applied to a moderate vector: every output entry is huge - // but representable. - Circulant<RealScalar> Ch(Vec(huge * c)); - Vec ones = Vec::Ones(n); - Vec z = Ch * ones; - VERIFY(z.allFinite()); - VERIFY(((z.array() - huge).abs() / huge).maxCoeff() <= kFftRoundTripTol); - - // Identity Toeplitz, same huge right-hand side. - Vec r = Vec::Zero(n); - r[0] = c[0]; - Toeplitz<RealScalar> T(c, r); - Vec w = T * x; - VERIFY(w.allFinite()); - VERIFY(((w - x).cwiseAbs() / huge).maxCoeff() <= kFftRoundTripTol); -} - -// The scaling exponents are derived from component-wise magnitudes: a finite -// complex value near the overflow threshold has a non-representable modulus, -// which would otherwise disable the scaling and turn an exactly representable -// product into NaN. -template <typename RealScalar> -void test_structured_fft_complex_boundary(Index n) { - typedef std::complex<RealScalar> Complex; - typedef Matrix<RealScalar, Dynamic, 1> RVec; - typedef Matrix<Complex, Dynamic, 1> CVec; - const RealScalar kFftRoundTripTol = RealScalar(100) * NumTraits<RealScalar>::epsilon(); - const RealScalar big = RealScalar(0.75) * (std::numeric_limits<RealScalar>::max)(); - - // Identity circulant with a complex generator: the product returns the - // right-hand side unchanged even though |x_k| overflows. - CVec c = CVec::Zero(n); - c[0] = Complex(1); - Circulant<Complex> C(c); - CVec x = CVec::Constant(n, Complex(big, big)); - CVec y = C * x; - VERIFY(y.allFinite()); - VERIFY(((y - x).cwiseAbs() / big).maxCoeff() <= kFftRoundTripTol); - - // A real identity operator applied to the same complex right-hand side takes - // the mixed-scalar product path. - RVec cr = RVec::Zero(n); - cr[0] = RealScalar(1); - Circulant<RealScalar> Cr(cr); - y = Cr * x; - VERIFY(y.allFinite()); - VERIFY(((y - x).cwiseAbs() / big).maxCoeff() <= kFftRoundTripTol); -} - -// A single Inf or NaN in the data must propagate like the reference product -- -// through the dot products that touch it -- instead of being smeared into NaNs -// across the whole output by the transforms. +// An Inf or NaN in the data is never lost and does not leak into other columns +// (see nonfinite_covers_reference()). template <typename Scalar> void test_structured_nonfinite_product(Index n) { typedef typename NumTraits<Scalar>::Real RealScalar; @@ -1035,50 +857,43 @@ Circulant<Scalar> C(c); Mat dense = reference_circulant<Scalar>(c); - // Inf in the right-hand side. Vec x = Vec::Random(n); x[n / 2] = Scalar(inf); - VERIFY(ieee_entrywise_match((C * x).eval(), reference_product_ieee(dense, x))); - - // NaN in the right-hand side. + VERIFY(nonfinite_covers_reference((C * x).eval(), reference_product_ieee(dense, x))); Vec xn = Vec::Random(n); xn[n - 1] = Scalar(nan); - VERIFY(ieee_entrywise_match((C * xn).eval(), reference_product_ieee(dense, xn))); + VERIFY(nonfinite_covers_reference((C * xn).eval(), reference_product_ieee(dense, xn))); - // Mixed multi-column right-hand side: the non-finite column falls back to the - // direct kernel individually while the finite column keeps the FFT path. Mat Xm(n, 2); Xm.col(0) = Vec::Random(n); Xm.col(1) = x; Mat Ym = C * Xm; VERIFY_IS_APPROX(Ym.col(0).eval(), (dense * Xm.col(0)).eval()); - VERIFY(ieee_entrywise_match(Ym.col(1).eval(), reference_product_ieee(dense, Vec(Xm.col(1))))); + VERIFY(nonfinite_covers_reference(Ym.col(1).eval(), reference_product_ieee(dense, Vec(Xm.col(1))))); - // Inf in the generator: the operator itself is non-finite, whatever the - // right-hand side. Vec c2 = Vec::Random(n); c2[1] = Scalar(-inf); Circulant<Scalar> C2(c2); Mat dense2 = reference_circulant<Scalar>(c2); Vec x2 = Vec::Random(n); - VERIFY(ieee_entrywise_match((C2 * x2).eval(), reference_product_ieee(dense2, x2))); + VERIFY(nonfinite_covers_reference((C2 * x2).eval(), reference_product_ieee(dense2, x2))); - // Toeplitz with an Inf in the row generator. Vec tc = Vec::Random(n), tr = Vec::Random(n); tr[0] = tc[0]; tr[n / 2] = Scalar(inf); Toeplitz<Scalar> T(tc, tr); Mat denseT = reference_toeplitz<Scalar>(tc, tr); - VERIFY(ieee_entrywise_match((T * x2).eval(), reference_product_ieee(denseT, x2))); + VERIFY(nonfinite_covers_reference((T * x2).eval(), reference_product_ieee(denseT, x2))); - // Non-finite right-hand sides of solve() take the direct pseudo-inverse - // application; on the 1x1 operator this is a single scalar multiply by the - // inverse coefficient, checked against the same multiply on the dense inverse. - Vec b1(1); - b1[0] = Scalar(inf); - Circulant<Scalar> C1(Vec(Vec::Constant(1, Scalar(2)))); - Mat pinv1 = Mat(C1.inverse()); - VERIFY(ieee_entrywise_match(C1.solve(b1), reference_product_ieee(pinv1, b1))); + Mat B(n, 2); + B.col(0) = Vec::Random(n); + B.col(1) = x; + Vec cd = Vec::Random(n); + cd[0] += Scalar(RealScalar(2 * n)); + Circulant<Scalar> Cd(cd); + Mat X = Cd.solve(B); + VERIFY_IS_APPROX((reference_circulant<Scalar>(cd) * X.col(0)).eval(), Vec(B.col(0))); + VERIFY(!X.col(1).allFinite()); } // Closed-form eigendecomposition: C * V = V * diag(eigenvalues) with V unitary. @@ -1219,6 +1034,7 @@ // A NaN in the generator must propagate to the solution (and count as non-zero in // rank()) instead of being silently laundered into zeros by the rank threshold. +template <typename = void> void test_circulant_nan_propagation(Index n) { typedef Matrix<double, Dynamic, 1> Vec; @@ -1269,6 +1085,7 @@ // plain product of the eigenvalues in FFT order overflows to infinity (or // underflows to an exact zero) partway through. Pins the balanced accumulation // in determinant(). +template <typename = void> void test_circulant_determinant_scaled() { typedef Matrix<double, Dynamic, 1> Vec; typedef Matrix<std::complex<double>, Dynamic, 1> CVec; @@ -1313,6 +1130,7 @@ // smallest-normal symbol entry is still inverted -- the comparison is strict, // matching SVDBase::rank(), which likewise reports rank one -- a subnormal entry // is treated as an exact zero, and non-finite entries count as non-zero. +template <typename = void> void test_circulant_rank_boundaries() { typedef Matrix<double, Dynamic, 1> Vec; const double mn = (std::numeric_limits<double>::min)(); @@ -1334,80 +1152,23 @@ VERIFY_IS_EQUAL(C.rank(), 1); VERIFY((numext::isinf)(C.determinant())); } -} - -// True where a subnormal result survives: under flush-to-zero -- NVHPC's default -// -fast, for one -- it is an exact zero instead. -bool subnormals_survive() { return !numext::is_exactly_zero(underflowProbe<double>()); } - -// A finite complex symbol entry near the overflow threshold has a -// non-representable modulus. The rank threshold used to be computed from the raw -// moduli, turning it into infinity: the rank was under-reported and solve() -// zeroed valid Fourier modes. Both are now evaluated in an exactly rescaled -// frame. -void test_circulant_rank_complex_boundary() { - typedef std::complex<double> Complex; - typedef Matrix<double, Dynamic, 1> Vec; - typedef Matrix<Complex, Dynamic, 1> CVec; - const double mx = (std::numeric_limits<double>::max)(); - - // The length-one shortcut has no FFTs, but its pointwise division still needs - // scaling: a huge right-hand side over the balanced O(1) symbol overflows before - // the symbol exponent is folded back, though the result itself is moderate. - const Vec oneCol = Vec::Constant(1, 0.75 * mx); - const Vec oneB = Vec::Constant(1, 0.25 * mx); - const Vec oneX = Circulant<double>(oneCol).solve(oneB); - const double oneExpected = oneB[0] / oneCol[0]; - VERIFY(oneX.allFinite()); - VERIFY(numext::abs(oneX[0] / oneExpected - 1.0) <= 16 * NumTraits<double>::epsilon()); - - // n = 2: the symbol is exactly [c0 + c1, c0 - c1], so pick the generator from - // the desired spectrum. |s0| overflows while both of its components are finite. - const Complex s0(0.75 * mx, 0.75 * mx), s1(1e300, 0.0); - CVec c(2); - c[0] = (s0 + s1) * 0.5; - c[1] = (s0 - s1) * 0.5; - Circulant<Complex> C(c); - VERIFY_IS_EQUAL(C.rank(), 2); - - // The second Fourier mode must be inverted, not zeroed: a product of a small - // vector solves back to that vector (the accuracy is limited by the condition - // number |s0| / |s1| ~ 2.5e8). - CVec x0(2); - x0[0] = Complex(1e-10, -2e-10); - x0[1] = Complex(-3e-10, 1e-10); - CVec b = C * x0; - VERIFY(b.allFinite()); - CVec x = C.solve(b); - VERIFY(((x - x0).cwiseAbs().maxCoeff() / x0.cwiseAbs().maxCoeff()) <= 1e-6); - - // A large right-hand side in the smaller retained mode must be scaled for the - // division's amplification as well as for the transforms: the quotient is - // moderate, but the balanced-frame division overflows without it. - const double h = 0.25 * mx; - CVec largeB(2); - largeB << Complex(h), Complex(-h); - const CVec largeX = C.solve(largeB); - const double expected = h / numext::real(C.symbol()[1]); - CVec expectedX(2); - expectedX << Complex(expected), Complex(-expected); - VERIFY(largeX.allFinite()); - VERIFY(((largeX - expectedX).cwiseAbs().maxCoeff() / expected) <= 32 * NumTraits<double>::epsilon()); - - // The reciprocal of an entry near the overflow boundary is subnormal but - // representable, and must not collapse to zero: formed as conj(z)/|z|^2, or by - // Smith's algorithm, it overflows to exactly zero and drops the mode. Gated on - // flush-to-zero, where that entry is not representable at all -- which is why - // solve() divides by the symbol rather than multiplying by its reciprocal. - const CVec sinv = C.inverse().symbol(); - VERIFY(sinv.allFinite()); - if (subnormals_survive()) VERIFY_IS_EQUAL((sinv.array() == Complex(0)).count(), 0); - - // A genuinely negligible second entry still truncates in the scaled frame. - CVec c2(2); - c2[0] = (s0 + Complex(1)) * 0.5; - c2[1] = (s0 - Complex(1)) * 0.5; - VERIFY_IS_EQUAL(Circulant<Complex>(c2).rank(), 1); + { + // Spectrum [s0, 1e300] with finite components but |s0| = Inf: the threshold + // stays finite, so the second mode is still inverted. + using Complex = std::complex<double>; + using CVec = Matrix<Complex, Dynamic, 1>; + const double mx = (std::numeric_limits<double>::max)(); + const Complex s0(0.75 * mx, 0.75 * mx), s1(1e300, 0.0); + CVec c(2); + c << (s0 + s1) * 0.5, (s0 - s1) * 0.5; + Circulant<Complex> C(c); + VERIFY_IS_EQUAL(C.rank(), 2); + CVec mode1(2); + mode1 << Complex(1), Complex(-1); // the second Fourier mode alone + CVec expected(2); + expected << Complex(1e-300), Complex(-1e-300); + VERIFY_IS_APPROX(C.solve(mode1), expected); + } } // Fixed-size operators: generators are stored in fixed-size vectors, products and @@ -1585,6 +1346,7 @@ // Indefinite / ill-conditioned matrices that force look-ahead block steps. The // generators and required block sizes are from Chan & Hansen's test set; the true // solution is the all-ones vector. +template <typename = void> void test_levinson_lookahead() { typedef Matrix<double, Dynamic, 1> Vec; // Loose bound for these deliberately ill-conditioned look-ahead cases (~1e-9); @@ -1634,6 +1396,7 @@ } // Fixed-size Toeplitz operators also feed the solver. +template <typename = void> void test_levinson_fixed() { typedef Matrix<double, 12, 1> Vec12; Vec12 c = Vec12::Random(), r = Vec12::Random(); @@ -1677,6 +1440,7 @@ } // A numerically singular Toeplitz must be reported through info(). +template <typename = void> void test_levinson_singular() { typedef Matrix<double, Dynamic, 1> Vec; for (Index n : {4, 9}) { @@ -1686,6 +1450,98 @@ } } +template <typename RealScalar> +bool structured_same_bits(const RealScalar& a, const RealScalar& b) { + using Binary = internal::binary_floating_point_traits<RealScalar>; + return Binary::bits(a) == Binary::bits(b); +} + +template <typename RealScalar> +bool structured_same_bits(const std::complex<RealScalar>& a, const std::complex<RealScalar>& b) { + return structured_same_bits(a.real(), b.real()) && structured_same_bits(a.imag(), b.imag()); +} + +template <typename Scalar> +struct structured_test_entry { + template <typename Real> + static Scalar run(Real real, Real) { + return real; + } + template <typename Real> + static Scalar ldexp(const Scalar& value, int e) { + return numext::ldexp(value, e); + } +}; + +template <typename Real> +struct structured_test_entry<std::complex<Real>> { + static std::complex<Real> run(Real real, Real imag) { return std::complex<Real>(real, imag); } + template <typename R> + static std::complex<Real> ldexp(const std::complex<Real>& value, int e) { + return std::complex<Real>(numext::ldexp(value.real(), e), numext::ldexp(value.imag(), e)); + } +}; + +// structured_exponent_bound() of an all-subnormal vector and +// structured_ldexp_entries() into and out of the subnormal range agree with +// frexp and ldexp bit for bit, in every flush-to-zero mode; the references +// are taken beforehand, where ldexp itself would flush. +template <typename Scalar> +void test_structured_flushed_subnormal_scaling() { + using Real = typename NumTraits<Scalar>::Real; + using Binary = internal::binary_floating_point_traits<Real>; + using Bits = typename Binary::Bits; + using Entry = structured_test_entry<Scalar>; + using Vec = Matrix<Scalar, Dynamic, 1>; + constexpr int digits = std::numeric_limits<Real>::digits; + constexpr int minExponent = std::numeric_limits<Real>::min_exponent; + const Index n = 7; + // Subnormal components 3 i + 5 and 2 i + 1 times denorm_min: the largest, 23 denorm_min, is below 2^5 denorm_min. + Vec tiny(n); + for (Index i = 0; i < n; ++i) + tiny(i) = Entry::run(numext::bit_cast<Real>(Bits(3 * i + 5)), numext::bit_cast<Real>(Bits(2 * i + 1))); + const int expectedBound = 5 + minExponent - digits + (NumTraits<Scalar>::IsComplex ? 1 : 0); + const int up = -expectedBound; + Vec expectedUp(n); + for (Index i = 0; i < n; ++i) expectedUp(i) = Entry::template ldexp<Real>(tiny(i), up); + const Vec normal = Vec::Random(n); + const int normalBound = internal::structured_exponent_bound(normal); + // Into the subnormal range: 2^(min_exponent - 5) times coefficients in [-1, 1] keeps digits - 5 bits at most. + const int down = minExponent - 5; + Vec expectedDown(n); + for (Index i = 0; i < n; ++i) expectedDown(i) = Entry::template ldexp<Real>(normal(i), down); + + // At the recovery threshold: the largest component 2^(recovery - 1) is normal, a component 2^(min_exponent - 2) + // relative 2^(1 - digits) below it is subnormal and must survive a scale-up by 2 (real and complex bounds alike). + const int recovery = internal::safe_scaling<Real>::subnormal_recovery_exponent(); + Vec boundary(2); + boundary(0) = Entry::run(numext::ldexp(Real(1), recovery - 1), Real(0)); + boundary(1) = Entry::run(numext::bit_cast<Real>(Binary::kExponentUnit >> 1), Real(0)); + Vec expectedBoundary(2); + for (Index i = 0; i < 2; ++i) expectedBoundary(i) = Entry::template ldexp<Real>(boundary(i), 1); + const int boundaryBound = internal::structured_exponent_bound(boundary); + + const auto check = [&]() { + int e = 0; + VERIFY(internal::structured_exponent_bound_finite(tiny, e)); + VERIFY_IS_EQUAL(e, expectedBound); + Vec doubled = boundary; + internal::structured_ldexp_entries(doubled, 1, boundaryBound); + for (Index i = 0; i < 2; ++i) VERIFY(structured_same_bits(doubled(i), expectedBoundary(i))); + VERIFY_IS_EQUAL(internal::structured_exponent_bound(tiny), expectedBound); + Vec scaled = tiny; + internal::structured_ldexp_entries(scaled, up, expectedBound); + for (Index i = 0; i < n; ++i) VERIFY(structured_same_bits(scaled(i), expectedUp(i))); + // The overload measuring its own input folds the exact scale-up back. + internal::structured_ldexp_entries(scaled, -up); + for (Index i = 0; i < n; ++i) VERIFY(structured_same_bits(scaled(i), tiny(i))); + Vec shrunk = normal; + internal::structured_ldexp_entries(shrunk, down, normalBound); + for (Index i = 0; i < n; ++i) VERIFY(structured_same_bits(shrunk(i), expectedDown(i))); + }; + forEachFlushToZeroMode([&](FlushToZeroMode) { check(); }); +} + EIGEN_DECLARE_TEST(structured_matrices) { for (int i = 0; i < g_repeat; ++i) { // Circulant: direct path (small), FFT path (composite and prime sizes), edge cases. @@ -1770,7 +1626,7 @@ CALL_SUBTEST_7((test_circulant_rank_deficient<std::complex<float>>(16, 2))); CALL_SUBTEST_7((test_circulant_zero<double>(12))); CALL_SUBTEST_7((test_circulant_zero<std::complex<double>>(7))); - CALL_SUBTEST_7(test_circulant_nan_propagation(20)); + CALL_SUBTEST_7(test_circulant_nan_propagation<>(20)); CALL_SUBTEST_7((test_circulant_inverse<double>(1))); CALL_SUBTEST_7((test_circulant_inverse<double>(50))); CALL_SUBTEST_7((test_circulant_inverse<double>(97))); // prime: inverse rebuilds the padded product symbol @@ -1799,14 +1655,8 @@ CALL_SUBTEST_8((test_circulant_mixed_scalar<float>(97))); CALL_SUBTEST_8((test_toeplitz_mixed_scalar<double>(8, 8))); CALL_SUBTEST_8((test_toeplitz_mixed_scalar<double>(64, 40))); - CALL_SUBTEST_8(test_circulant_determinant_scaled()); - CALL_SUBTEST_8(test_circulant_rank_boundaries()); - CALL_SUBTEST_8(test_circulant_rank_complex_boundary()); - CALL_SUBTEST_8((test_structured_fft_overflow<double>(40))); - CALL_SUBTEST_8((test_structured_fft_overflow<double>(97))); // prime, no 5-smooth padding - CALL_SUBTEST_8((test_structured_fft_overflow<float>(40))); - CALL_SUBTEST_8((test_structured_fft_complex_boundary<double>(40))); - CALL_SUBTEST_8((test_structured_fft_complex_boundary<float>(40))); + CALL_SUBTEST_8(test_circulant_determinant_scaled<>()); + CALL_SUBTEST_8(test_circulant_rank_boundaries<>()); // Aliasing beyond the same-object case, across the dispatch tiers. CALL_SUBTEST_8((test_structured_aliased_expression<double>(8))); @@ -1814,8 +1664,7 @@ CALL_SUBTEST_8((test_structured_aliased_expression<double>(48))); CALL_SUBTEST_8((test_structured_aliased_expression<std::complex<double>>(40))); - // Entrywise Inf/NaN propagation: FFT-sized operators must fall back to the - // direct kernel; small ones are IEEE-exact already. + // Inf/NaN propagation on the direct and FFT tiers. CALL_SUBTEST_8((test_structured_nonfinite_product<double>(40))); CALL_SUBTEST_8((test_structured_nonfinite_product<double>(12))); CALL_SUBTEST_8((test_structured_nonfinite_product<std::complex<double>>(40))); @@ -1831,9 +1680,9 @@ CALL_SUBTEST_11((test_levinson_wellcond<float>(40))); CALL_SUBTEST_12((test_levinson_wellcond<std::complex<double>>(30))); CALL_SUBTEST_12((test_levinson_solve_in_expression<std::complex<double>>(16))); - CALL_SUBTEST_13(test_levinson_lookahead()); - CALL_SUBTEST_13(test_levinson_fixed()); - CALL_SUBTEST_13(test_levinson_singular()); + CALL_SUBTEST_13(test_levinson_lookahead<>()); + CALL_SUBTEST_13(test_levinson_fixed<>()); + CALL_SUBTEST_13(test_levinson_singular<>()); CALL_SUBTEST_13((test_levinson_solve_in_expression<double>(1))); CALL_SUBTEST_13((test_levinson_solve_in_expression<double>(24))); CALL_SUBTEST_14((test_levinson_wellcond<std::complex<float>>(24))); @@ -1877,7 +1726,7 @@ CALL_SUBTEST_9((test_hankel_solve<float>(16))); CALL_SUBTEST_9((test_hankel_solve_lookahead<double>(20))); CALL_SUBTEST_9((test_hankel_solve_lookahead<std::complex<double>>(15))); - CALL_SUBTEST_9(test_hankel_hilbert()); + CALL_SUBTEST_9(test_hankel_hilbert<>()); CALL_SUBTEST_9((test_hankel_least_squares<double>(60, 40))); CALL_SUBTEST_9((test_hankel_least_squares<std::complex<double>>(48, 32))); CALL_SUBTEST_9((test_hankel_fixed<double, 4, 6>())); @@ -1886,12 +1735,8 @@ CALL_SUBTEST_9((test_hankel_fixed_solve<double, 8>())); CALL_SUBTEST_9((test_hankel_fixed_solve<std::complex<double>, 12>())); - // MR 2688 review regressions: finite-overflow scaling in the FFT tier, - // delayed (value-nested) products, aliased right-hand sides across the + // MR 2688 review regressions: delayed (value-nested) products, aliased right-hand sides across the // dispatch tiers, and mixed real/complex products. - CALL_SUBTEST_10(test_hankel_fft_overflow()); - CALL_SUBTEST_10((test_hankel_fft_complex_boundary<double>(40))); - CALL_SUBTEST_10((test_hankel_fft_complex_boundary<float>(40))); CALL_SUBTEST_10((test_hankel_delayed_product<double>(24, 12))); CALL_SUBTEST_10((test_hankel_delayed_product<double>(40, 64))); CALL_SUBTEST_10((test_hankel_delayed_product<std::complex<double>>(48, 33))); @@ -1906,8 +1751,7 @@ CALL_SUBTEST_10((test_hankel_aliased_expression<double>(48))); CALL_SUBTEST_10((test_hankel_aliased_expression<std::complex<double>>(40))); - // Entrywise Inf/NaN propagation: FFT-sized operators must fall back to the - // direct kernels; small ones are IEEE-exact already. + // Inf/NaN propagation on the direct and FFT tiers. CALL_SUBTEST_10((test_hankel_nonfinite_product<double>(40))); CALL_SUBTEST_10((test_hankel_nonfinite_product<double>(12))); CALL_SUBTEST_10((test_hankel_nonfinite_product<std::complex<double>>(40))); @@ -1918,5 +1762,10 @@ CALL_SUBTEST_10((test_hankel_mixed_scalar<double>(1, 40))); // skinny direct paths CALL_SUBTEST_10((test_hankel_mixed_scalar<double>(40, 1))); CALL_SUBTEST_10((test_hankel_mixed_scalar<float>(48, 64))); + + CALL_SUBTEST_15((test_structured_flushed_subnormal_scaling<float>())); + CALL_SUBTEST_15((test_structured_flushed_subnormal_scaling<double>())); + CALL_SUBTEST_15((test_structured_flushed_subnormal_scaling<std::complex<float>>())); + CALL_SUBTEST_15((test_structured_flushed_subnormal_scaling<std::complex<double>>())); } }
diff --git a/contrib/test/structured_test_helpers.h b/contrib/test/structured_test_helpers.h new file mode 100644 index 0000000..ef5dc9b --- /dev/null +++ b/contrib/test/structured_test_helpers.h
@@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#ifndef EIGEN_STRUCTURED_TEST_HELPERS_H +#define EIGEN_STRUCTURED_TEST_HELPERS_H + +#include <Eigen/Core> + +namespace Eigen { + +// Vectorized complex products can introduce Inf - Inf across component accumulators. +// Preserve scalar evaluation order for the non-finite reference results. +template <typename Scalar> +Matrix<Scalar, Dynamic, 1> reference_product_ieee(const Matrix<Scalar, Dynamic, Dynamic>& matrix, + const Matrix<Scalar, Dynamic, 1>& vector) { + Matrix<Scalar, Dynamic, 1> result(matrix.rows()); + for (Index i = 0; i < matrix.rows(); ++i) { + Scalar accumulator(0); + for (Index j = 0; j < matrix.cols(); ++j) accumulator += matrix(i, j) * vector[j]; + result[i] = accumulator; + } + return result; +} + +// The FFT-based products mix every input entry into every output entry, so an Inf or NaN may make more entries +// non-finite than the dense product does. \returns whether it makes at least those. +template <typename Actual, typename Reference> +bool nonfinite_covers_reference(const Actual& actual, const Reference& reference) { + for (Index i = 0; i < reference.size(); ++i) + if (!(numext::isfinite)(reference[i]) && (numext::isfinite)(actual[i])) return false; + return true; +} + +} // namespace Eigen + +#endif // EIGEN_STRUCTURED_TEST_HELPERS_H
diff --git a/contrib/test/structured_vandermonde.cpp b/contrib/test/structured_vandermonde.cpp index aea01a2..271ae0d 100644 --- a/contrib/test/structured_vandermonde.cpp +++ b/contrib/test/structured_vandermonde.cpp
@@ -132,6 +132,7 @@ // Eigen::half has explicit conversion from the wider type returned by the // standard scaling functions. Instantiate both product and solver paths so the // range-protection helpers retain support for narrow floating-point scalars. +template <typename = void> void test_vandermonde_half() { typedef Matrix<half, 2, 1> Vec; Vec nodes, coefficients; @@ -152,6 +153,7 @@ // Björck-Pereyra's celebrated accuracy property (Higham, ASNA ch. 22): for // monotone nodes and an alternating-sign right-hand side the forward error is // tiny even though the matrix conditioning is astronomical. Deterministic. +template <typename = void> void test_bjorck_pereyra_higham() { typedef Matrix<double, Dynamic, 1> Vec; @@ -192,6 +194,7 @@ // The transpose and adjoint recurrences must be distinguished for genuinely // complex nodes. A rotated root-of-unity grid keeps both systems well // conditioned while avoiding the unrotated grid's extra symmetries. +template <typename = void> void test_bjorck_pereyra_complex_transpose_adjoint(Index n) { typedef std::complex<double> Scalar; typedef Matrix<double, Dynamic, 1> RealVec; @@ -248,6 +251,7 @@ // Repeated nodes make the matrix exactly singular, while non-finite nodes make // the solver input invalid. Both must be reported before solve evaluation. +template <typename = void> void test_bjorck_pereyra_singular() { typedef Matrix<double, Dynamic, 1> Vec; Vec x(5); @@ -392,22 +396,16 @@ } // Reference for the wide-dynamic-range determinant tests: the same factor sequence with explicit -// exponent tracking, halving any factor whose node difference overflows (exact, since a difference -// only overflows for huge normal operands). Power-of-two rescaling is exact, so this reproduces the -// exact product up to one rounding per multiplication, representable where naive partial products -// leave the double range. Finite nodes only. +// exponent tracking. Power-of-two rescaling is exact, so this reproduces the exact product up to one +// rounding per multiplication, representable where naive partial products leave the double range. +// Finite node differences only. double reference_vandermonde_det(const Matrix<double, Dynamic, 1>& x) { double mantissa = 1.0; Index exponent = 0; for (Index j = 1; j < x.size(); ++j) for (Index i = 0; i < j; ++i) { - double diff = x[j] - x[i]; - if (!(numext::isfinite)(diff)) { - diff = 0.5 * x[j] - 0.5 * x[i]; - ++exponent; - } int e; - mantissa *= std::frexp(diff, &e); + mantissa *= std::frexp(x[j] - x[i], &e); exponent += e; mantissa = std::frexp(mantissa, &e); exponent += e; @@ -419,6 +417,7 @@ // product is kept as mantissa * 2^e with exact frexp/ldexp renormalization, so // partial products that leave the representable range cannot destroy a // representable determinant, while zeros and genuine overflow still propagate. +template <typename = void> void test_vandermonde_determinant_scaled() { typedef Matrix<double, Dynamic, 1> Vec; typedef std::complex<double> Complex; @@ -505,165 +504,13 @@ } } -// Reviewer reproducer on MR 2691: a node difference can overflow while the determinant stays -// representable. For nodes [-DBL_MAX, DBL_MAX, d, 2d, ..., 5d] with d the smallest subnormal, -// DBL_MAX - (-DBL_MAX) = 2^1025 overflows, yet det = -576 DBL_MAX^11 d^10 ~ -3.1633e160. The -// overflowing difference must be halved (exact for these huge normal operands) and the factor of -// two carried in the running exponent. -void test_vandermonde_determinant_overflowing_differences() { +// Horner's rule matches a naive scalar Horner loop to its forward error bound (Higham, ASNA 2nd ed., +// section 5.1): |p(x) - fl(p(x))| <= gamma_{2n} sum_j |a_j| |x|^j <= 2n eps sum_j |a_j| for |x| <= 1. +// A genuinely unrepresentable value saturates. +template <typename = void> +void test_vandermonde_horner() { typedef Matrix<double, Dynamic, 1> Vec; - typedef std::complex<double> Complex; - const double M = (std::numeric_limits<double>::max)(); - const double d = std::numeric_limits<double>::denorm_min(); - const double kBalancedTol = 16 * NumTraits<double>::epsilon(); - const double kAnalyticTol = 100 * NumTraits<double>::epsilon(); - - Vec x(7); - x << -M, M, d, 2 * d, 3 * d, 4 * d, 5 * d; - const double det = Vandermonde<double>(x).determinant(); - VERIFY((numext::isfinite)(det)); - - // Scaled reference computation with the identical factor sequence. - const double ref = reference_vandermonde_det(x); - VERIFY(numext::abs(det / ref - 1.0) <= kBalancedTol); - - // Analytic factored form, accumulated with the same exponent bookkeeping: - // every huge factor rounds to +-DBL_MAX (and the halved leading factor is - // exactly DBL_MAX * 2), the d-spaced block contributes exactly 288 * 2^-10740, - // so det = -2 * 288 * DBL_MAX^11 * 2^-10740. - double mantissa = -576.0; - Index exponent = -10740; - for (int t = 0; t < 11; ++t) { - int e; - mantissa *= std::frexp(M, &e); - exponent += e; - mantissa = std::frexp(mantissa, &e); - exponent += e; - } - const double expected = std::ldexp(mantissa, static_cast<int>(exponent)); - VERIFY(numext::abs(det / expected - 1.0) <= kAnalyticTol); - VERIFY(numext::abs(det / -3.1633e160 - 1.0) <= 1e-3); // the reviewer's quoted value - - // Complex nodes i*x: every factor becomes i*(x_j - x_i) -- the overflow now - // sits in the imaginary components, which the component-wise finiteness check - // must catch -- and det picks up i^21 = i. - Matrix<Complex, Dynamic, 1> xc = Complex(0, 1) * x.cast<Complex>(); - const Complex detc = Vandermonde<Complex>(xc).determinant(); - const Complex refc = Complex(0, 1) * Complex(ref); - VERIFY((numext::isfinite)(detc)); - VERIFY(numext::abs(detc - refc) <= kBalancedTol * numext::abs(refc)); -} - -// Reviewer reproducer on MR 2691: Horner intermediates can overflow while the value stays -// representable. At node 1/2 with coefficients [0, DBL_MAX, DBL_MAX] the intermediate 1.5 DBL_MAX -// is Inf, but the value is exactly fl(0.75 DBL_MAX). The scaled path must return it bit-exactly, -// the plain path must match the naive loop for moderate data, and genuinely unrepresentable values -// must still saturate. -void test_vandermonde_scaled_horner() { - typedef Matrix<double, Dynamic, 1> Vec; - typedef std::complex<double> Complex; - typedef Matrix<Complex, Dynamic, 1> CVec; - const double M = (std::numeric_limits<double>::max)(); - { - // The reviewer's exact reproducer, in each accumulation form. - Vec x(1); - x << 0.5; - Vandermonde<double> V(x, 3); - Vec a(3); - a << 0.0, M, M; - Vec y = V * a; - VERIFY_IS_EQUAL(y[0], 0.75 * M); - y.setZero(); - y.noalias() += V * a; - VERIFY_IS_EQUAL(y[0], 0.75 * M); - // Two identical columns exercise the per-column screening. - Matrix<double, Dynamic, Dynamic> A(3, 2); - A.col(0) = a; - A.col(1) = a; - Matrix<double, Dynamic, Dynamic> Y = V * A; - VERIFY_IS_EQUAL(Y(0, 0), 0.75 * M); - VERIFY_IS_EQUAL(Y(0, 1), 0.75 * M); - } - { - // Complex path: the same coefficients rotated by i keep the overflow in a - // single component; the value is i * fl(0.75 * DBL_MAX). - CVec xc(1); - xc << Complex(0.5, 0.0); - Vandermonde<Complex> Vc(xc, 3); - CVec ac(3); - ac << Complex(0), Complex(0, M), Complex(0, M); - CVec yc = Vc * ac; - VERIFY_IS_EQUAL(numext::real(yc[0]), 0.0); - VERIFY_IS_EQUAL(numext::imag(yc[0]), 0.75 * M); - } - { - // Genuine overflow must still saturate: value 3 * DBL_MAX at the node 2. - Vec x(1); - x << 2.0; - Vandermonde<double> V(x, 2); - Vec a(2); - a << M, M; - Vec y = V * a; - VERIFY((numext::isinf)(y[0]) && y[0] > 0.0); - a << M, -M; // value -DBL_MAX: representable again, sign preserved - y = V * a; - VERIFY_IS_EQUAL(y[0], -M); - } - { - // A vanished accumulator has no scale (reviewer repro): after the running - // value becomes exactly zero -- a zero node annihilating it, or an exact - // cancellation -- the frame exponent must reset, or the trailing small - // coefficient underflows in the stale huge frame and the value 0 is - // returned instead of the coefficient itself. - const double mn = (std::numeric_limits<double>::min)(); - const double dm = std::numeric_limits<double>::denorm_min(); - Vec x(1); - x << 0.0; // zero node: the value is exactly a[0] - Vandermonde<double> V0(x, 3); - Vec a(3); - a << mn, M, M; - Vec y = V0 * a; - VERIFY_IS_EQUAL(y[0], mn); - - x << 1.0; // exact cancellation: M - M + dm = dm - Vandermonde<double> V1(x, 3); - a << dm, -M, M; - y = V1 * a; - VERIFY_IS_EQUAL(y[0], dm); - } - { - // Complex variant: the real components cancel exactly while the tiny - // imaginary coefficient must survive in a fresh frame. - const double dm = std::numeric_limits<double>::denorm_min(); - CVec xc(1); - xc << Complex(1.0, 0.0); - Vandermonde<Complex> Vc(xc, 3); - CVec ac(3); - ac << Complex(0.0, dm), Complex(-M, 0.0), Complex(M, 0.0); - CVec yc = Vc * ac; - VERIFY_IS_EQUAL(numext::real(yc[0]), 0.0); - VERIFY_IS_EQUAL(numext::imag(yc[0]), dm); - } - { - // Near-cancellation: M - (M - 2^971) leaves the tiny nonzero intermediate - // 2^971 (mantissa 2^-53 in the huge frame), which the frexp renormalization - // must rebase; the trailing coefficient 1 then rounds away exactly as in - // real arithmetic: fl(2^971 + 1) = 2^971. - const double big = std::ldexp(1.0, 971); // ulp(DBL_MAX), and M - big is exact - Vec x(1); - x << 1.0; - Vandermonde<double> V(x, 3); - Vec a(3); - a << 1.0, -(M - big), M; - Vec y = V * a; - VERIFY_IS_EQUAL(y[0], big); - } - { - // Moderate data keeps the plain path, so this matches a naive Horner loop to Horner's own forward - // error bound (Higham, ASNA 2nd ed., section 5.1): |p(x) - fl(p(x))| <= gamma_{2n} sum_j |a_j| |x|^j - // <= 2n eps sum_j |a_j| for |x| <= 1. Not bit-identical, since the compiler may contract - // acc * x[i] + a[j] into an FMA in one of the two loops and not the other. const Index m = 7, n = 6; Vec x = Vec::Random(m), a = Vec::Random(n); Vandermonde<double> V(x, n); @@ -675,6 +522,35 @@ VERIFY(numext::abs(y[i] - acc) <= bound); } } + { + const double M = (std::numeric_limits<double>::max)(); + Vec x(1); + x << 2.0; + Vandermonde<double> V(x, 2); + Vec a(2); + a << M, M; // value 3 * DBL_MAX + Vec y = V * a; + VERIFY((numext::isinf)(y[0]) && y[0] > 0.0); + } + { + // Complex values at real nodes keep real-times-complex arithmetic: an infinite real part, from the + // data or from overflow, leaves the imaginary part exactly zero as in a dense product. + typedef std::complex<double> Complex; + typedef Matrix<Complex, Dynamic, 1> CVec; + const double inf = std::numeric_limits<double>::infinity(); + Vec x(2); + x << 2.0, 3.0; + CVec a(2); + a << Complex(1), Complex(inf, 0); + CVec y = Vandermonde<double>(x, 2) * a; + for (Index i = 0; i < 2; ++i) VERIFY((numext::isinf)(numext::real(y[i])) && numext::imag(y[i]) == 0.0); + Vec x1(1); + x1 << 10.0; + CVec a3(3); + a3 << Complex(1), Complex(1), Complex(1e308); + CVec y3 = Vandermonde<double>(x1, 3) * a3; + VERIFY((numext::isinf)(numext::real(y3[0])) && numext::imag(y3[0]) == 0.0); + } } template <typename Scalar, int M, int N> @@ -699,6 +575,7 @@ // Core rewrites alpha * (V * a) as (alpha * V) * a. Keep the two scaled // expression forms together because they share the same structured wrapper. +template <typename = void> void test_vandermonde_expression_regressions() { typedef Matrix<double, 1, 1> Vec1; typedef Matrix<double, 3, 1> Vec3; @@ -725,6 +602,7 @@ VERIFY_IS_APPROX((V * (B * C)).eval(), (dense * (B * C)).eval()); } +template <typename = void> void test_vandermonde_dimension_checks() { VectorXd x(2); x << 0.0, 1.0; @@ -738,6 +616,7 @@ // returned from factories remain alive. Their node storage must therefore avoid // over-alignment: generic Product and cwise wrapper types do not supply aligned // operator new under C++14. +template <typename = void> void test_vandermonde_fixed_heap_expressions() { typedef Matrix<double, 4, 1> Vec4; typedef Matrix<double, 4, 4> Mat4; @@ -783,22 +662,21 @@ CALL_SUBTEST_2((test_bjorck_pereyra<std::complex<double>>(10))); CALL_SUBTEST_2((test_bjorck_pereyra_roots_of_unity<double>(64))); CALL_SUBTEST_2((test_bjorck_pereyra_roots_of_unity<float>(32))); - CALL_SUBTEST_2(test_vandermonde_half()); - CALL_SUBTEST_2(test_bjorck_pereyra_higham()); - CALL_SUBTEST_2(test_bjorck_pereyra_complex_transpose_adjoint(12)); - CALL_SUBTEST_2(test_bjorck_pereyra_singular()); + CALL_SUBTEST_2(test_vandermonde_half<>()); + CALL_SUBTEST_2(test_bjorck_pereyra_higham<>()); + CALL_SUBTEST_2(test_bjorck_pereyra_complex_transpose_adjoint<>(12)); + CALL_SUBTEST_2(test_bjorck_pereyra_singular<>()); // Closed-form determinant and fixed sizes. CALL_SUBTEST_3((test_vandermonde_determinant<double>(8))); CALL_SUBTEST_3((test_vandermonde_determinant<std::complex<double>>(7))); - CALL_SUBTEST_3(test_vandermonde_determinant_scaled()); - CALL_SUBTEST_3(test_vandermonde_determinant_overflowing_differences()); - CALL_SUBTEST_3(test_vandermonde_scaled_horner()); + CALL_SUBTEST_3(test_vandermonde_determinant_scaled<>()); + CALL_SUBTEST_3(test_vandermonde_horner<>()); CALL_SUBTEST_3((test_vandermonde_fixed<double, 6, 4>())); CALL_SUBTEST_3((test_vandermonde_fixed<double, 5, 5>())); CALL_SUBTEST_3((test_vandermonde_fixed<std::complex<float>, 4, 4>())); - CALL_SUBTEST_3(test_vandermonde_expression_regressions()); - CALL_SUBTEST_3(test_vandermonde_fixed_heap_expressions()); + CALL_SUBTEST_3(test_vandermonde_expression_regressions<>()); + CALL_SUBTEST_3(test_vandermonde_fixed_heap_expressions<>()); // Product regressions: aliasing, operand lifetime, mixed scalars. CALL_SUBTEST_4((test_vandermonde_aliased_product<double>(8))); @@ -812,5 +690,5 @@ CALL_SUBTEST_4((test_vandermonde_mixed_scalar<float>(7, 9))); } - CALL_SUBTEST_3(test_vandermonde_dimension_checks()); + CALL_SUBTEST_3(test_vandermonde_dimension_checks<>()); }
diff --git a/contrib/test/structured_vandermonde_int_index.cpp b/contrib/test/structured_vandermonde_int_index.cpp index 4d9cc1b..ee10514 100644 --- a/contrib/test/structured_vandermonde_int_index.cpp +++ b/contrib/test/structured_vandermonde_int_index.cpp
@@ -29,23 +29,7 @@ VERIFY(det > 0.0); } -// The scaled-Horner frame can likewise exceed a 32-bit Index exponent. A -// degree-2,099,999 monomial at DBL_MAX must overflow to +Inf rather than wrap -// its accumulated exponent and underflow to zero. -void test_vandermonde_scaled_horner_int_index() { - typedef Matrix<double, Dynamic, 1> Vec; - const Index n = 2100000; - Vec x(1), a = Vec::Zero(n); - x[0] = (std::numeric_limits<double>::max)(); - a[n - 1] = 1.0; - - Vec y = Vandermonde<double>(x, n) * a; - VERIFY((numext::isinf)(y[0])); - VERIFY(y[0] > 0.0); -} - EIGEN_DECLARE_TEST(structured_vandermonde_int_index) { STATIC_CHECK(sizeof(Index) == sizeof(int)); CALL_SUBTEST_1(test_vandermonde_determinant_int_index()); - CALL_SUBTEST_2(test_vandermonde_scaled_horner_int_index()); }
diff --git a/contrib/test/tensor_gpu.cu b/contrib/test/tensor_gpu.cu index 6689473..1c4480c 100644 --- a/contrib/test/tensor_gpu.cu +++ b/contrib/test/tensor_gpu.cu
@@ -140,6 +140,41 @@ gpuFree(d_out); } +// Integer exponents on aligned maps take the packet path, where the GPU packets lack the double-word squaring +// operations and must keep plain squaring. +template <typename T> +void test_gpu_pow_integer_exponent() { + Tensor<T, 1> in(Eigen::array<Eigen::DenseIndex, 1>{64}); + Tensor<T, 1> out(Eigen::array<Eigen::DenseIndex, 1>{64}); + in.setRandom(); + in = in * T(0.5) + T(1.25); + + std::size_t bytes = in.size() * sizeof(T); + T* d_in; + T* d_out; + gpuMalloc((void**)(&d_in), bytes); + gpuMalloc((void**)(&d_out), bytes); + gpuMemcpy(d_in, in.data(), bytes, gpuMemcpyHostToDevice); + + Eigen::GpuStreamDevice stream; + Eigen::GpuDevice gpu_device(&stream); + Eigen::TensorMap<Eigen::Tensor<T, 1>, Eigen::Aligned> gpu_in(d_in, Eigen::array<Eigen::DenseIndex, 1>{64}); + Eigen::TensorMap<Eigen::Tensor<T, 1>, Eigen::Aligned> gpu_out(d_out, Eigen::array<Eigen::DenseIndex, 1>{64}); + + for (int exponent : {3, 8, -2}) { + gpu_out.device(gpu_device) = gpu_in.pow(exponent); + assert(gpuMemcpyAsync(out.data(), d_out, bytes, gpuMemcpyDeviceToHost, gpu_device.stream()) == gpuSuccess); + assert(gpuStreamSynchronize(gpu_device.stream()) == gpuSuccess); + for (int i = 0; i < 64; ++i) { + VERIFY_IS_APPROX(out(Eigen::array<Eigen::DenseIndex, 1>{i}), + T(std::pow(double(in(Eigen::array<Eigen::DenseIndex, 1>{i})), double(exponent)))); + } + } + + gpuFree(d_in); + gpuFree(d_out); +} + void test_gpu_elementwise() { Tensor<float, 3> in1(Eigen::array<Eigen::DenseIndex, 3>{72, 53, 97}); Tensor<float, 3> in2(Eigen::array<Eigen::DenseIndex, 3>{72, 53, 97}); @@ -1549,6 +1584,8 @@ CALL_SUBTEST_1((test_gpu_nullary_max_size<int64_t, (std::numeric_limits<int32_t>::max)() + 100ll>())); CALL_SUBTEST_1(test_gpu_elementwise_small()); CALL_SUBTEST_1(test_gpu_elementwise()); + CALL_SUBTEST_1(test_gpu_pow_integer_exponent<float>()); + CALL_SUBTEST_1(test_gpu_pow_integer_exponent<double>()); CALL_SUBTEST_1(test_gpu_props()); CALL_SUBTEST_1(test_gpu_striding<ColMajor>()); CALL_SUBTEST_1(test_gpu_striding<RowMajor>());
diff --git a/contrib/test/tensor_random.cpp b/contrib/test/tensor_random.cpp index 45b1afc..17b7b7c 100644 --- a/contrib/test/tensor_random.cpp +++ b/contrib/test/tensor_random.cpp
@@ -71,6 +71,15 @@ } template <typename Scalar> +static void test_normal_zero_uniform_draw() { + // Regression test: element 2429913123 of seed 12345 draws u == 0 (probability 2^-23 per float draw) with q between + // 0.27597 and 0.27846, where the ratio-of-uniforms loop reached the NaN log test and returned v / 0 = -inf. The + // element index is a uint64_t because it exceeds a 32-bit Index. + Eigen::internal::NormalRandomGenerator<Scalar> generator(12345); + VERIFY((numext::isfinite)(generator(numext::uint64_t(2429913123u)))); +} + +template <typename Scalar> static void test_uniform_range(Eigen::Index size) { // All uniform draws must lie in [0, 1). Tensor<Scalar, 1> vec(size); @@ -206,6 +215,8 @@ CALL_SUBTEST((test_normal<Eigen::bfloat16>())); CALL_SUBTEST((test_normal_all_finite<Eigen::half>(Eigen::Index(1) << 21))); CALL_SUBTEST((test_normal_all_finite<Eigen::bfloat16>(Eigen::Index(1) << 18))); + CALL_SUBTEST(test_normal_zero_uniform_draw<float>()); + CALL_SUBTEST(test_normal_zero_uniform_draw<Eigen::half>()); CALL_SUBTEST((test_uniform_range<Eigen::half>(Eigen::Index(1) << 16))); CALL_SUBTEST((test_uniform_range<Eigen::bfloat16>(Eigen::Index(1) << 16))); CALL_SUBTEST((test_block_materialization<float, Eigen::internal::UniformRandomGenerator<float>>()));
diff --git a/contrib/test/tensor_thread_pool.cpp b/contrib/test/tensor_thread_pool.cpp index 3f6863d..f30c4f2 100644 --- a/contrib/test/tensor_thread_pool.cpp +++ b/contrib/test/tensor_thread_pool.cpp
@@ -37,6 +37,7 @@ int dealloc_count_ = 0; }; +template <typename = void> void test_multithread_elementwise() { Tensor<float, 3> in1(200, 30, 70); Tensor<float, 3> in2(200, 30, 70); @@ -58,6 +59,7 @@ } } +template <typename = void> void test_async_multithread_elementwise() { Tensor<float, 3> in1(200, 30, 70); Tensor<float, 3> in2(200, 30, 70); @@ -82,6 +84,7 @@ } } +template <typename = void> void test_multithread_chip() { Tensor<float, 5> in(2, 3, 5, 7, 11); Tensor<float, 4> out(3, 5, 7, 11); @@ -104,6 +107,7 @@ } } +template <typename = void> void test_async_multithread_chip() { Tensor<float, 5> in(2, 3, 5, 7, 11); Tensor<float, 4> out(3, 5, 7, 11); @@ -128,6 +132,7 @@ } } +template <typename = void> void test_multithread_volume_patch() { Tensor<float, 5> in(4, 2, 3, 5, 7); Tensor<float, 6> out(4, 1, 1, 1, 2 * 3 * 5, 7); @@ -144,6 +149,7 @@ } } +template <typename = void> void test_async_multithread_volume_patch() { Tensor<float, 5> in(4, 2, 3, 5, 7); Tensor<float, 6> out(4, 1, 1, 1, 2 * 3 * 5, 7); @@ -185,6 +191,7 @@ // the argument through coeff() with BlockAccess independent of the argument, // so these tests exercise the tiled executor's per-task evaluator copy // regardless of how any one evaluator's traits evolve. +template <typename = void> void test_multithread_tiled_stateful_functor() { Tensor<float, 4> in(64, 64, 64, 8); in.setRandom(); @@ -206,6 +213,7 @@ } } +template <typename = void> void test_async_multithread_tiled_stateful_functor() { Tensor<float, 4> in(64, 64, 64, 8); in.setRandom(); @@ -229,6 +237,7 @@ } } +template <typename = void> void test_multithread_compound_assignment() { Tensor<float, 3> in1(2, 3, 7); Tensor<float, 3> in2(2, 3, 7); @@ -763,6 +772,7 @@ VERIFY_IS_APPROX(full_redux(), full_redux_tp()); } +template <typename = void> void test_multithreaded_complex_reduction() { using Scalar = std::complex<float>; constexpr Index size = 4096; @@ -782,6 +792,7 @@ VERIFY_IS_EQUAL(output(), Scalar(static_cast<float>(size), static_cast<float>(2 * size))); } +template <typename = void> void test_multithreaded_complex_partial_reductions() { using Scalar = std::complex<float>; constexpr Index outer_size = 32; @@ -812,6 +823,7 @@ } } +template <typename = void> void test_memcpy() { for (int i = 0; i < 5; ++i) { const int num_threads = internal::random<int>(3, 11); @@ -829,6 +841,7 @@ } } +template <typename = void> void test_multithread_random() { Eigen::ThreadPool tp(2); Eigen::ThreadPoolDevice device(&tp, 2); @@ -860,6 +873,7 @@ } } +template <typename = void> void test_threadpool_allocate(TestAllocator* allocator) { const int num_threads = internal::random<int>(2, 11); const int num_allocs = internal::random<int>(2, 11); @@ -1133,22 +1147,22 @@ } EIGEN_DECLARE_TEST(tensor_thread_pool) { - CALL_SUBTEST_1(test_multithread_elementwise()); - CALL_SUBTEST_1(test_async_multithread_elementwise()); - CALL_SUBTEST_1(test_multithread_compound_assignment()); + CALL_SUBTEST_1(test_multithread_elementwise<>()); + CALL_SUBTEST_1(test_async_multithread_elementwise<>()); + CALL_SUBTEST_1(test_multithread_compound_assignment<>()); CALL_SUBTEST_2(test_multithread_contraction<ColMajor>()); CALL_SUBTEST_2(test_multithread_contraction<RowMajor>()); CALL_SUBTEST_2(test_multithread_gemv_transpose<ColMajor>()); CALL_SUBTEST_2(test_multithread_gemv_transpose<RowMajor>()); - CALL_SUBTEST_3(test_multithread_chip()); - CALL_SUBTEST_3(test_async_multithread_chip()); + CALL_SUBTEST_3(test_multithread_chip<>()); + CALL_SUBTEST_3(test_async_multithread_chip<>()); - CALL_SUBTEST_4(test_multithread_volume_patch()); - CALL_SUBTEST_4(test_async_multithread_volume_patch()); - CALL_SUBTEST_4(test_multithread_tiled_stateful_functor()); - CALL_SUBTEST_4(test_async_multithread_tiled_stateful_functor()); + CALL_SUBTEST_4(test_multithread_volume_patch<>()); + CALL_SUBTEST_4(test_async_multithread_volume_patch<>()); + CALL_SUBTEST_4(test_multithread_tiled_stateful_functor<>()); + CALL_SUBTEST_4(test_async_multithread_tiled_stateful_functor<>()); CALL_SUBTEST_5(test_multithread_contraction_agrees_with_singlethread<ColMajor>()); CALL_SUBTEST_5(test_multithread_contraction_agrees_with_singlethread<RowMajor>()); @@ -1178,16 +1192,16 @@ CALL_SUBTEST_11(test_multithreaded_reductions<ColMajor>()); CALL_SUBTEST_11(test_multithreaded_reductions<RowMajor>()); - CALL_SUBTEST_11(test_multithreaded_complex_reduction()); - CALL_SUBTEST_11(test_multithreaded_complex_partial_reductions()); + CALL_SUBTEST_11(test_multithreaded_complex_reduction<>()); + CALL_SUBTEST_11(test_multithreaded_complex_partial_reductions<>()); - CALL_SUBTEST_12(test_memcpy()); - CALL_SUBTEST_12(test_multithread_random()); + CALL_SUBTEST_12(test_memcpy<>()); + CALL_SUBTEST_12(test_multithread_random<>()); TestAllocator test_allocator; CALL_SUBTEST_13(test_multithread_shuffle<ColMajor>(nullptr)); CALL_SUBTEST_13(test_multithread_shuffle<RowMajor>(&test_allocator)); - CALL_SUBTEST_13(test_threadpool_allocate(&test_allocator)); + CALL_SUBTEST_13(test_threadpool_allocate<>(&test_allocator)); CALL_SUBTEST_14(test_multithread_zero_dim_contraction<ColMajor>()); CALL_SUBTEST_14(test_multithread_zero_dim_contraction<RowMajor>());
diff --git a/doc/PreprocessorDirectives.dox b/doc/PreprocessorDirectives.dox index b12db06..cc7f710 100644 --- a/doc/PreprocessorDirectives.dox +++ b/doc/PreprocessorDirectives.dox
@@ -110,11 +110,26 @@ - \b \c EIGEN_UNROLLING_LIMIT - defines the size of a loop to enable meta unrolling. Set it to zero to disable unrolling. The size of a loop here is expressed in %Eigen's own notion of "number of FLOPS", it does not correspond to the number of iterations or the number of instructions. The default is value 110. + - \b \c EIGEN_GEMM_TO_COEFFBASED_THRESHOLD - a matrix-matrix product that reaches the general (GEMM) product path + falls back at run time to the coefficient-based product when rows + cols + depth is below this value. Default is 20. + In an ARM SME build, the scalar types the SME GEMM kernel handles use \c EIGEN_SME_GEMM_TO_COEFFBASED_THRESHOLD + instead when it is defined, and also fall back when the result is not a vector and has at most + \c EIGEN_SME_GEMM_TO_COEFFBASED_OUTPUT_AREA_THRESHOLD(Scalar) coefficients (default 96 / sizeof(Scalar)). + - \b \c EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD - a product that would take the GEMM path and whose three + dimensions are all fixed at compile time uses the coefficient-based product instead, selected at compile time, + when rows + cols + depth is below this value. Default is 40. It does not follow + \c EIGEN_GEMM_TO_COEFFBASED_THRESHOLD, which still applies at run time to the fixed-size products this one sends to + the GEMM path, so their crossover is the larger of the two values. + - \b \c EIGEN_SME_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD - the same compile-time bound for the scalar types the ARM + SME GEMM kernel handles (\c float, \c std::complex<float>, and with FEAT_SME_F64F64 \c double and + \c std::complex<double>). Default is 61, measured on Apple M4 Pro; other scalar types use + \c EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD in an SME build too. - \b \c EIGEN_STACK_ALLOCATION_LIMIT - defines the maximum bytes for a buffer to be allocated on the stack. For internal temporary buffers, dynamic memory allocation is employed as a fall back; setting the limit to 0 disables stack allocation for them, so every such temporary is heap-allocated. For fixed-size matrices or arrays, exceeding this threshold raises a compile time assertion; there, 0 disables the check. Default is 128 KB. - - \b \c EIGEN_STACK_ALIGN_BYTES - alignment of the internal temporary buffers allocated on the stack. Defaults to + - \b \c EIGEN_STACK_ALIGN_BYTES - alignment of the internal temporary buffers: the ones on the stack, their heap + fallback above \c EIGEN_STACK_ALLOCATION_LIMIT and the GEMM blocking buffers. Defaults to \c EIGEN_DEFAULT_ALIGN_BYTES, except in ARM SME builds where it is 64: the SME GEMM kernel reads its packed panels a streaming vector at a time and slows by 35-50% when they straddle 64-byte lines. It does not affect the ABI. - \b \c EIGEN_NO_CUDA - disables CUDA support when defined. Might be useful in .cu files for which Eigen is used on the host only, @@ -135,14 +150,37 @@ and multiplies a block with NEON instead of the ZA tiles (panels no deeper than the depth bound and no wider than the panel bound; a block of such panels no wider than the width bound, or the shallow width up to the shallow depth, or narrower than the thin bound on one side). One value overrides the per-scalar defaults, which are fitted - for Apple M4. \c EIGEN_SME_NO_NEON_SMALL_BLOCKS and \c EIGEN_SME_FORCE_NEON_SMALL_BLOCKS pin the path for A/B runs. - - \b \c EIGEN_ARM64_NO_SME_F64F64 - Disables the double-precision ARM SME GEMM kernel, which needs the optional + for Apple M4. \c EIGEN_SME_NO_NEON_SMALL_BLOCKS and \c EIGEN_SME_FORCE_NEON_SMALL_BLOCKS pin the path for A/B runs; + both also turn off the NEON kernel for results of at most 8 x 8 (float) or 4 x 8 (double). + - \b \c EIGEN_SME_UNITS - Number of ARM SME units a product on the SME GEMM kernel spreads over; more threads than + units only contend for them. Detected from the core topology by default (see \c Eigen::nbSmeUnits() and + \c Eigen::setNbSmeUnits()): on macOS from the performance clusters, and 0 (no cap) elsewhere. Define it on other + systems whose SME units are shared by a cluster of cores, such as Linux on Apple silicon. + - \b \c EIGEN_SME_MIN_TASK_SIZE - Minimum work per thread, in FP32 multiply-adds, before a product on the ARM SME + GEMM kernel is split across threads. Other types count their kernel time in FP32 multiply-adds: double and + complex<float> four each, complex<double> sixteen. The default (2^20) is tuned for Apple M4 and applies where the + SME unit count is known (see \c EIGEN_SME_UNITS). There, when the maximum sizes of the result and of the inner + dimension are not all fixed at compile time, each thread computes a disjoint part of the result. + - \b \c EIGEN_SME_DIRECT_LHS_MAX_STRIDE_BYTES and \b \c EIGEN_SME_DIRECT_LHS_MAX_BLOCK_BYTES - Largest column stride + and largest block (rows times depth, both in bytes) at which the ARM SME GEMM kernel reads a column-major real LHS + block from its source instead of a packed panel, whatever its alignment. The defaults are 16 KB and 4 MB; strides + that are a multiple of 4 KB always pack, since they alias in L1. 0 for either disables the in-place read. These + limits do not apply when the RHS is a single panel (see \c EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES). + - \b \c EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES - When the RHS is a single panel, each LHS element is read once, so the + ARM SME GEMM kernel reads a column-major real LHS in place: always up to half a panel wide, and up to a full panel + while depth times column stride (in bytes) stays within this limit, past which the panel spans more pages than + the TLB covers. The default is 16 MB; 0 disables the in-place read for single-panel products. + - \b \c EIGEN_ARM64_NO_SME_F64F64 - Disables the double-precision ARM SME kernels, which need the optional FEAT_SME_F64F64 extension. %Eigen enables it whenever the compiler reports that extension; define this macro when - the build target is wider than the run target, so that \c double keeps the generic kernel. + the build target is wider than the run target, so that \c double keeps the generic kernels for matrix products, + matrix-vector products, dot products, and scaled vector additions. - \b \c EIGEN_SME_PACKED_RHS_BUDGET_BYTES - Byte budget for the packed RHS panel in the ARM SME GEMM kernel. The default (32 MB) is empirically tuned for Apple M4. - \b \c EIGEN_SME_LHS_WORKING_SET_BUDGET_BYTES - Byte budget for the LHS working set in the ARM SME GEMM kernel. The default (7 MB) is empirically tuned for Apple M4. + - \b \c EIGEN_SME_SINGLE_PASS_RHS_BUDGET_BYTES - Byte budget for the packed RHS panel in the ARM SME GEMM kernel when + all rows of the product fit one LHS block, so that each packed panel is read once and should stay in L2 between + packing and the kernel. The default (4 MB) is empirically tuned for Apple M4 (16 MB L2 per cluster). - \b \c EIGEN_DONT_ALIGN - Deprecated, it is a synonym for \c EIGEN_MAX_ALIGN_BYTES=0. It disables alignment completely. %Eigen will not try to align its objects and does not expect that any objects passed to it are aligned. This will turn off vectorization if \b \c EIGEN_UNALIGNED_VECTORIZE=1. Not defined by default. - \b \c EIGEN_DONT_ALIGN_STATICALLY - Deprecated, it is a synonym for \c EIGEN_MAX_STATIC_ALIGN_BYTES=0. It disables alignment of arrays on the stack. Not defined by default, unless \c EIGEN_DONT_ALIGN is defined.
diff --git a/doc/QuickReference.dox b/doc/QuickReference.dox index 8d1ee24..91f006a 100644 --- a/doc/QuickReference.dox +++ b/doc/QuickReference.dox
@@ -653,6 +653,14 @@ \endcode</td> </tr> +<tr><td>Lazy sums with dense matrices and with triangular views (no dense copy of the diagonal is formed)</td> + <td>\code +mat3 = mat1 + diag1; +mat3 = vec1.asDiagonal() - mat1; +mat3 = mat1.triangularView<Eigen::Lower>() + diag1; // still a triangular view +\endcode</td> +</tr> + </table> \subsection QuickRef_TriangularView Triangular views
diff --git a/doc/TopicLinearAlgebraDecompositions.dox b/doc/TopicLinearAlgebraDecompositions.dox index 8fb7bb7..2280c67 100644 --- a/doc/TopicLinearAlgebraDecompositions.dox +++ b/doc/TopicLinearAlgebraDecompositions.dox
@@ -337,7 +337,7 @@ <ul> <li><a name="note1">\b 1: </a>There exist two variants of the LDLT algorithm. %Eigen's LDLT produces a pure diagonal D matrix and therefore cannot handle indefinite matrices; it uses symmetric pivoting, which keeps the factorization stable when pivots are tiny or zero. The other variant, which produces the block-diagonal D needed for indefinite self-adjoint matrices, is provided by the BunchKaufman class.</li> <li><a name="note2">\b 2: </a>Eigenvalues, SVD and Schur decompositions rely on iterative algorithms. Their convergence speed depends on how well the eigenvalues are separated.</li> -<li><a name="note3">\b 3: </a>Our JacobiSVD is two-sided, which buys high relative accuracy on square matrices: where a bidiagonalizing algorithm only computes each singular value accurately relative to the largest one, Jacobi computes it accurately relative to itself, so that tiny singular values are still meaningful. That property is conditional rather than absolute -- the relative error is governed by the condition number of the matrix obtained by scaling its columns to unit norm, so a matrix that stays ill-conditioned under any such scaling loses it (J. Demmel and K. Veselic, "Jacobi's method is more accurate than QR", SIAM J. Matrix Anal. Appl. 13 (1992), pp. 1204-1245). For non-square matrices, we have to use a QR preconditioner first. Householder QR is backward stable with or without pivoting, so this choice is about rank revelation rather than about the accuracy of the factorization itself: the default \c ColPivHouseholderQRPreconditioner reveals rank reliably in practice but without a guarantee, \c FullPivHouseholderQRPreconditioner inspects the whole trailing submatrix at a substantially higher cost, and \c HouseholderQRPreconditioner and \c NoQRPreconditioner do not pivot at all. See the guidance below on complete pivoting.</li> +<li><a name="note3">\b 3: </a>Our JacobiSVD is two-sided, which buys high relative accuracy on square matrices: where a bidiagonalizing algorithm only computes each singular value accurately relative to the largest one, Jacobi computes it accurately relative to itself, so that tiny singular values are still meaningful. That property is conditional rather than absolute -- the relative error is governed by the condition number of the matrix obtained by scaling its columns to unit norm, so a matrix that stays ill-conditioned under any such scaling loses it (J. Demmel and K. Veselic, "Jacobi's method is more accurate than QR", SIAM J. Matrix Anal. Appl. 13 (1992), pp. 1204-1245). For non-square matrices, we have to use a QR preconditioner first; \c PreconditionSquareMatrix applies it to square matrices too. Householder QR is backward stable with or without pivoting, so this choice is about rank revelation rather than about the accuracy of the factorization itself: the default \c ColPivHouseholderQRPreconditioner reveals rank reliably in practice but without a guarantee, \c FullPivHouseholderQRPreconditioner inspects the whole trailing submatrix at a substantially higher cost, and \c HouseholderQRPreconditioner and \c NoQRPreconditioner do not pivot at all. See the guidance below on complete pivoting.</li> <li><a name="note4">\b 4: </a>These factorizations are all backward stable, but pivoting buys something different in each. For LU the backward error bound is proportional to the growth factor: partial pivoting bounds it only by \f$ 2^{n-1} \f$, whereas complete pivoting admits a far smaller bound that grows only slightly faster than polynomially in \c n. That partial-pivoting worst case is not observed on real matrices, however, and partial pivoting is more than sufficient in practice. Householder QR is backward stable irrespective of pivoting, with a bound that involves no growth factor at all; there, pivoting serves rank revelation rather than stability, and what separates column from complete pivoting is how reliably the trailing diagonal of R exposes a rank deficiency. Since complete pivoting also rules out the cache-friendly blocked algorithms, LAPACK and %Eigen both treat partial and column pivoting as the defaults for LU and QR; see the guidance below. This is a judgement about those two factorizations rather than about complete pivoting in general: LAPACK does provide a complete-pivoting Cholesky (\c pstrf) for positive semidefinite matrices, where the pivoting also determines the rank.</li> <li><a name="note5">\b 5: </a>Cholesky needs no pivoting for stability: on a positive definite matrix the growth factor is bounded by 1, so the unpivoted factorization is backward stable as it stands. What conditioning governs is instead whether it completes: on a matrix that is positive definite but nearly singular, rounding can drive a computed pivot to zero or below, and %Eigen reports that through info() rather than returning a meaningless factor. Use LDLT, or a rank-revealing decomposition, for such matrices.</li> </ul>
diff --git a/doc/TopicMultithreading.dox b/doc/TopicMultithreading.dox index 1cc189e..c8e593f 100644 --- a/doc/TopicMultithreading.dox +++ b/doc/TopicMultithreading.dox
@@ -17,6 +17,10 @@ \endcode Unless `setNbThreads` has been called, %Eigen uses the number of threads specified by OpenMP. You can restore this behavior by calling `setNbThreads(0);`. +On the ARM SME backend, a product that runs on the SME GEMM kernel uses at most `Eigen::nbSmeUnits()` threads, +one per SME unit (the units are shared by a cluster of cores on Apple silicon; macOS reports them, other systems set +the count with `EIGEN_SME_UNITS` or `Eigen::setNbSmeUnits(n)`), each computing a disjoint part of the result; +`setNbSmeUnits(0)` removes the cap and returns to the shared parallel session. You can query the number of threads that will be used with: \code n = Eigen::nbThreads( );
diff --git a/doc/TopicVectorization.dox b/doc/TopicVectorization.dox index f3b4cb7..da7e4ee 100644 --- a/doc/TopicVectorization.dox +++ b/doc/TopicVectorization.dox
@@ -20,7 +20,8 @@ <tr class="alt"><td>ARM / AArch64</td><td>NEON, SME; SVE as an opt-in backend</td> <td>SVE requires \c EIGEN_ARM64_USE_SVE and a fixed vector length (<tt>-msve-vector-bits=N</tt>). The SME backend accelerates real and complex matrix - products and is selected automatically when the compiler targets SME2 + products as well as large real dot products, scaled vector additions and column-major + matrix-vector products. It is selected automatically when the compiler targets SME2 (<tt>-march=armv9.2-a+sme2</tt>, or a <tt>-mcpu</tt> that implies it) \em without <tt>-msve-vector-bits</tt>, since a fixed length would pin the kernels to one runtime streaming vector length; everything else keeps the NEON path. Define @@ -28,7 +29,10 @@ cannot provide it into a build error instead of a NEON fallback. Double precision additionally needs the optional FEAT_SME_F64F64 extension (<tt>+sme-f64f64</tt>, or a <tt>-mcpu</tt> that implies it); without it \c double and \c std::complex<double> keep - the generic kernel</td></tr> + the generic kernels. Floating-point exceptions raised in the streaming SME kernels (matrix + products, and the vector and matrix-vector kernels) are not reported: FMOPA sets no flags, + and the caller's flags are restored on exit, discarding those of the kernels' other + arithmetic. Products on the NEON paths report exceptions as usual</td></tr> <tr><td>PowerPC</td><td>AltiVec, VSX, MMA</td><td></td></tr> <tr class="alt"><td>IBM Z (s390x)</td><td>ZVector</td><td></td></tr> <tr><td>MIPS</td><td>MSA</td><td></td></tr>
diff --git a/failtest/CMakeLists.txt b/failtest/CMakeLists.txt index afa179a..57c4bd3 100644 --- a/failtest/CMakeLists.txt +++ b/failtest/CMakeLists.txt
@@ -59,6 +59,7 @@ ei_add_failtest("ternary_1") ei_add_failtest("ternary_2") +ei_add_failtest("approx_mixed_scalars") ei_add_failtest("sparse_ref_1") ei_add_failtest("sparse_ref_2") @@ -76,6 +77,9 @@ ei_add_failtest("colpivqr_int") ei_add_failtest("fullpivqr_int") ei_add_failtest("jacobisvd_int") +ei_add_failtest("jacobisvd_fullpiv_thinu") +ei_add_failtest("jacobisvd_fullpiv_thinv") +ei_add_failtest("jacobisvd_precondition_square_matrix_noqr") ei_add_failtest("bdcsvd_int") ei_add_failtest("eigensolver_int") ei_add_failtest("eigensolver_cplx")
diff --git a/failtest/approx_mixed_scalars.cpp b/failtest/approx_mixed_scalars.cpp new file mode 100644 index 0000000..68b8b53 --- /dev/null +++ b/failtest/approx_mixed_scalars.cpp
@@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include "../Eigen/Core" + +int main() { + const Eigen::Matrix<Eigen::half, 3, 1> a = Eigen::Matrix<Eigen::half, 3, 1>::Ones(); + const Eigen::Vector3f b = Eigen::Vector3f::Ones(); +#ifdef EIGEN_SHOULD_FAIL_TO_BUILD + return int(a.isApprox(b)); +#else + return int(a.cast<float>().isApprox(b)); +#endif +}
diff --git a/failtest/jacobisvd_fullpiv_thinu.cpp b/failtest/jacobisvd_fullpiv_thinu.cpp new file mode 100644 index 0000000..3c4d3f9 --- /dev/null +++ b/failtest/jacobisvd_fullpiv_thinu.cpp
@@ -0,0 +1,11 @@ +#include "../Eigen/SVD" + +#ifdef EIGEN_SHOULD_FAIL_TO_BUILD +#define U_OPTION ComputeThinU +#else +#define U_OPTION ComputeFullU +#endif + +using namespace Eigen; + +int main() { JacobiSVD<MatrixXf, FullPivHouseholderQRPreconditioner | U_OPTION> svd(MatrixXf::Random(6, 4)); }
diff --git a/failtest/jacobisvd_fullpiv_thinv.cpp b/failtest/jacobisvd_fullpiv_thinv.cpp new file mode 100644 index 0000000..37c03ed --- /dev/null +++ b/failtest/jacobisvd_fullpiv_thinv.cpp
@@ -0,0 +1,11 @@ +#include "../Eigen/SVD" + +#ifdef EIGEN_SHOULD_FAIL_TO_BUILD +#define V_OPTION ComputeThinV +#else +#define V_OPTION ComputeFullV +#endif + +using namespace Eigen; + +int main() { JacobiSVD<MatrixXf, FullPivHouseholderQRPreconditioner | V_OPTION> svd(MatrixXf::Random(4, 6)); }
diff --git a/failtest/jacobisvd_precondition_square_matrix_noqr.cpp b/failtest/jacobisvd_precondition_square_matrix_noqr.cpp new file mode 100644 index 0000000..c8beb06 --- /dev/null +++ b/failtest/jacobisvd_precondition_square_matrix_noqr.cpp
@@ -0,0 +1,11 @@ +#include "../Eigen/SVD" + +#ifdef EIGEN_SHOULD_FAIL_TO_BUILD +#define QR_OPTION NoQRPreconditioner +#else +#define QR_OPTION ColPivHouseholderQRPreconditioner +#endif + +using namespace Eigen; + +int main() { JacobiSVD<MatrixXf, QR_OPTION | PreconditionSquareMatrix> svd(MatrixXf::Random(4, 4)); }
diff --git a/failtest/permutation_unsupported_shape.cpp b/failtest/permutation_unsupported_shape.cpp index 58b77f0..33bd0a0 100644 --- a/failtest/permutation_unsupported_shape.cpp +++ b/failtest/permutation_unsupported_shape.cpp
@@ -9,7 +9,7 @@ permutation.setIdentity(); Eigen::Matrix2d matrix = Eigen::Matrix2d::Identity(); #ifdef EIGEN_SHOULD_FAIL_TO_BUILD - auto operand = matrix.triangularView<Eigen::Lower>(); + Eigen::internal::BandMatrix<double, 2, 2, 1, 1> operand; #else auto operand = matrix; #endif
diff --git a/lapack/dsecnd_NONE.f b/lapack/dsecnd_NONE.f new file mode 100644 index 0000000..518d4aa --- /dev/null +++ b/lapack/dsecnd_NONE.f
@@ -0,0 +1,54 @@ +*> \brief \b DSECND returns nothing +* +* =========== DOCUMENTATION =========== +* +* Online html documentation available at +* http://www.netlib.org/lapack/explore-html/ +* +* Definition: +* =========== +* +* DOUBLE PRECISION FUNCTION DSECND( ) +* +* +*> \par Purpose: +* ============= +*> +*> \verbatim +*> +*> DSECND returns nothing instead of returning the user time for a process in seconds. +*> If you are using that routine, it means that neither EXTERNAL ETIME, +*> EXTERNAL ETIME_, INTERNAL ETIME, INTERNAL CPU_TIME is available on +*> your machine. +*> \endverbatim +* +* Authors: +* ======== +* +*> \author Univ. of Tennessee +*> \author Univ. of California Berkeley +*> \author Univ. of Colorado Denver +*> \author NAG Ltd. +* +*> \date November 2011 +* +*> \ingroup auxOTHERauxiliary +* +* ===================================================================== +*> SPDX-FileCopyrightText: The Eigen Authors +*> SPDX-License-Identifier: BSD-3-Clause + DOUBLE PRECISION FUNCTION DSECND( ) +* +* -- LAPACK auxiliary routine (version 3.4.0) -- +* -- LAPACK is a software package provided by Univ. of Tennessee, -- +* -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..-- +* November 2011 +* +* ===================================================================== +* + DSECND = 0.0D+0 + RETURN +* +* End of DSECND +* + END
diff --git a/lapack/second_NONE.f b/lapack/second_NONE.f new file mode 100644 index 0000000..7c0d793 --- /dev/null +++ b/lapack/second_NONE.f
@@ -0,0 +1,54 @@ +*> \brief \b SECOND returns nothing +* +* =========== DOCUMENTATION =========== +* +* Online html documentation available at +* http://www.netlib.org/lapack/explore-html/ +* +* Definition: +* =========== +* +* REAL FUNCTION SECOND( ) +* +* +*> \par Purpose: +* ============= +*> +*> \verbatim +*> +*> SECOND returns nothing instead of returning the user time for a process in seconds. +*> If you are using that routine, it means that neither EXTERNAL ETIME, +*> EXTERNAL ETIME_, INTERNAL ETIME, INTERNAL CPU_TIME is available on +*> your machine. +*> \endverbatim +* +* Authors: +* ======== +* +*> \author Univ. of Tennessee +*> \author Univ. of California Berkeley +*> \author Univ. of Colorado Denver +*> \author NAG Ltd. +* +*> \date November 2011 +* +*> \ingroup auxOTHERauxiliary +* +* ===================================================================== +*> SPDX-FileCopyrightText: The Eigen Authors +*> SPDX-License-Identifier: BSD-3-Clause + REAL FUNCTION SECOND( ) +* +* -- LAPACK auxiliary routine (version 3.4.0) -- +* -- LAPACK is a software package provided by Univ. of Tennessee, -- +* -- Univ. of California Berkeley, Univ. of Colorado Denver and NAG Ltd..-- +* November 2011 +* +* ===================================================================== +* + SECOND = 0.0E+0 + RETURN +* +* End of SECOND +* + END
diff --git a/scripts/affected_tests.py b/scripts/affected_tests.py index 65f8490..1a99707 100644 --- a/scripts/affected_tests.py +++ b/scripts/affected_tests.py
@@ -114,6 +114,10 @@ # and force the full suite anyway. "ci/*.gitlab-ci.yml", "ci/CTest2JUnit.xsl", + # The clang-tidy and lint images run no test; the build and test images + # are in ci/docker/. + "ci/tidy/*", + "ci/lint/*", "debug/*", "demos/*", "doc/*",
diff --git a/scripts/ci_cmake_msvc.ps1 b/scripts/ci_cmake_msvc.ps1 index 9b20103..feb17bf 100644 --- a/scripts/ci_cmake_msvc.ps1 +++ b/scripts/ci_cmake_msvc.ps1
@@ -24,7 +24,7 @@ IF (!$EIGEN_CI_CMAKE_GENERATOR) { $EIGEN_CI_CMAKE_GENERATOR = "Ninja" } IF (!$EIGEN_CI_MSVC_ARCH) { $EIGEN_CI_MSVC_ARCH = "x64" } IF (!$EIGEN_CI_MSVC_VER) { $EIGEN_CI_MSVC_VER = "14.29" } -IF (!$EIGEN_CI_TEST_CUSTOM_CXX_FLAGS) { $EIGEN_CI_TEST_CUSTOM_CXX_FLAGS = "/d2ReducedOptimizeHugeFunctions /DEIGEN_STRONG_INLINE=inline /Os" } +IF (!$EIGEN_CI_TEST_CUSTOM_CXX_FLAGS) { $EIGEN_CI_TEST_CUSTOM_CXX_FLAGS = "-d2ReducedOptimizeHugeFunctions -DEIGEN_STRONG_INLINE=inline -Os" } # Export variables into the global scope $global:EIGEN_CI_ROOTDIR = $EIGEN_CI_ROOTDIR
diff --git a/scripts/test_affected_tests.py b/scripts/test_affected_tests.py index 05e5988..a6d302f 100644 --- a/scripts/test_affected_tests.py +++ b/scripts/test_affected_tests.py
@@ -156,7 +156,8 @@ # merely dropping them from FULL_REBUILD_PATTERNS would leave them falling # through to "not in the include graph" and forcing the full suite anyway. for path in ["ci/test.linux.gitlab-ci.yml", "ci/common.gitlab-ci.yml", - "ci/build.linux.gitlab-ci.yml", "ci/CTest2JUnit.xsl", "ci/README.md"]: + "ci/build.linux.gitlab-ci.yml", "ci/CTest2JUnit.xsl", "ci/README.md", + "ci/tidy/Dockerfile", "ci/lint/Dockerfile"]: sel = select(graph, [path]) check(sel.mode == "none", "%s selects nothing, got %s (%s)" % (path, sel.mode, sel.reasons)) @@ -463,21 +464,26 @@ "real tree has many test sources, got %d" % len(source_targets)) # Manual add_executable targets are named explicitly in full-suite selection. - # redux_bounded_compile is also a dependency of redux_bounded; bug1213 and + # Both redux_bounded_compile targets are dependencies of redux_bounded; bug1213 and # ulp_accuracy have no aggregate. Asserted as an exact set, because a # name that reaches this set without belonging in it makes the build jobs' # "not configured in this build" diagnostic permanently non-empty. - check(registered.standalone == {"bug1213", "redux_bounded_compile", "ulp_accuracy"}, + compile_targets = {"redux_bounded_compile", "redux_bounded_compile_vectorized"} + check(registered.standalone == {"bug1213", "ulp_accuracy"} | compile_targets, "unexpected standalone target set, got %s" % sorted(registered.standalone)) if "test/bug1213.cpp" in graph.files: check("\nbug1213\n" in full_suite(graph, []).targets_file, "full mode names bug1213, got %r" % full_suite(graph, []).targets_file) - check("redux_bounded_compile" in full_suite(graph, []).targets_file.splitlines(), - "full mode names redux_bounded_compile") + check(compile_targets <= set(full_suite(graph, []).targets_file.splitlines()), + "full mode names both bounded compile targets") sel = select(graph, ["test/redux_bounded_compile.cpp"]) - check(sel.mode == "targets" and sel.targets == {"redux_bounded_compile"}, - "compile regression source selects its target, got %s (%s)" + check(sel.mode == "targets" and sel.targets == compile_targets, + "compile regression source selects both targets, got %s (%s)" + % (sorted(sel.targets), sel.mode)) + sel = select(graph, ["test/redux_bounded_compile_vectorized.cpp"]) + check(sel.mode == "targets" and sel.targets == {"redux_bounded_compile_vectorized"}, + "compile regression wrapper selects only its target, got %s (%s)" % (sorted(sel.targets), sel.mode)) # test/buildsystem/ is under a test root but is not part of this build.
diff --git a/scripts/test_tidy_compile_db.py b/scripts/test_tidy_compile_db.py index 9a80dd6..e3f4b0e 100644 --- a/scripts/test_tidy_compile_db.py +++ b/scripts/test_tidy_compile_db.py
@@ -23,23 +23,32 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from style_common import REPO_ROOT # noqa: E402 -from tidy_compile_db import (MAX_PARTS, added_lines, configuration_of, part_of, # noqa: E402 +from tidy_compile_db import (MAX_PARTS, added_lines, configuration_of, parts_of, # noqa: E402 reduce_entries, required_parts, summarize) SCRIPT = os.path.join(REPO_ROOT, "scripts", "tidy_compile_db.py") def entry(source, part=None, extra=(), output="obj"): - """A compilation-database entry the way CMake's Ninja generator writes one.""" - flags = " ".join(list(extra) + (["-DEIGEN_TEST_PART_%s=1" % part] if part else [])) + """A compilation-database entry the way CMake's Ninja generator writes one. + + ``part`` is one part, or the list of parts a grouped range defines. + """ + parts = [part] if isinstance(part, str) else list(part or ()) + flags = " ".join(list(extra) + ["-DEIGEN_TEST_PART_%s=1" % member for member in parts]) return {"directory": "/build", "file": source, "command": "/usr/bin/c++ %s -I/repo -std=c++14 -o %s -c %s" % (flags, output, source), "output": output} +def part_label(kept): + """The parts an entry defines, as "3" or, for a grouped range, "1,2,3".""" + return ",".join(sorted(parts_of(kept), key=int)) + + def parts_kept(matches, text, lines): keep, _, _ = reduce_entries(matches, text, lines) - return [part_of(kept) for kept in keep] + return [part_label(kept) for kept in keep] def test_part_define_does_not_distinguish_configurations(): @@ -47,7 +56,7 @@ first = entry("/repo/test/t.cpp", part="1", output="t_1.o") second = entry("/repo/test/t.cpp", part="2", output="t_2.o") assert configuration_of(first) == configuration_of(second) - assert part_of(first) == "1" and part_of(second) == "2" + assert part_label(first) == "1" and part_label(second) == "2" # A macro that is not a part define does distinguish them, and so does no # define at all. shared = entry("/repo/blas/b.cpp", extra=["-DEIGEN_BLAS_BUILD_DLL", "-fPIC"]) @@ -138,6 +147,28 @@ assert required_parts(text, [3], frozenset(["1", "2", "3"])) == [frozenset(["1"])] +def test_grouped_range_covers_each_of_its_parts(): + """A grouped range is one entry defining every part in it. + + Keying an entry on its first part define would leave parts 2 and 3 to no + entry, and a line inside `CALL_SUBTEST_3` would go unchecked. + """ + matches = [entry("/repo/test/t.cpp", part=["1", "2", "3"], output="t_1.o"), + entry("/repo/test/t.cpp", part="4", output="t_4.o")] + text = ("void run() {\n" + " CALL_SUBTEST_3(f());\n" + " CALL_SUBTEST_4(g());\n" + "}\n") + keep, skipped, unreachable = reduce_entries(matches, text, [2]) + assert [part_label(kept) for kept in keep] == ["1,2,3"] and not skipped and not unreachable + assert "checking parts 1, 2, 3" in summarize(matches, keep, skipped, unreachable) + assert parts_kept(matches, text, [2, 3]) == ["1,2,3", "4"] + assert parts_kept(matches, text, [3]) == ["4"] + # Entries that share a part are not one configuration split into parts. + overlapping = matches + [entry("/repo/test/t.cpp", part="3", output="t_3.o")] + assert parts_kept(overlapping, text, [3]) == ["1,2,3", "4", "3"] + + def test_cap_reports_the_parts_it_leaves_out(): count = MAX_PARTS + 2 matches = [entry("/repo/test/t.cpp", part=str(n), output="t_%d.o" % n) @@ -187,7 +218,7 @@ " CALL_SUBTEST_2(f());\n" "}\n") keep, _, _ = reduce_entries(matches, text, [2]) - assert [(part_of(kept), kept["output"]) for kept in keep] == [("2", "plain_2.o"), + assert [(part_label(kept), kept["output"]) for kept in keep] == [("2", "plain_2.o"), ("2", "novec_2.o")] @@ -224,7 +255,7 @@ assert done.returncode == 0, done with open(os.path.join(outdir, "compile_commands.json")) as handle: reduced = json.load(handle) - assert [part_of(kept) for kept in reduced] == ["3"], reduced + assert [part_label(kept) for kept in reduced] == ["3"], reduced assert "checking part 3" in done.stdout, done.stdout # A source outside the database is reported by exit status, not output.
diff --git a/scripts/tidy_compile_db.py b/scripts/tidy_compile_db.py index bb39a7a..6e5f64b 100644 --- a/scripts/tidy_compile_db.py +++ b/scripts/tidy_compile_db.py
@@ -6,8 +6,9 @@ ``ci/scripts/run-clang-tidy.sh`` checks a changed source through the build directory's ``compile_commands.json``. clang-tidy parses the file once per -entry naming it, and a split test contributes one entry per ``EIGEN_TEST_PART`` --- 17 for ``test/eigensolver_selfadjoint.cpp``, 41 for ``test/array_cwise.cpp``, +entry naming it, and a split test contributes one entry per ``EIGEN_TEST_PART``, +or per range of parts ``cmake/EigenTestPartGroups.cmake`` compiles together +-- 17 for ``test/eigensolver_selfadjoint.cpp``, 14 for ``test/array_cwise.cpp``, at roughly a minute and 3 GB each. A merge request touching such a file spends the job's entire timeout on it, and every file after it goes unchecked without saying so. @@ -47,8 +48,8 @@ import shlex import sys -# A part costs about a minute and 2 GB, against a 15 minute job budget shared -# with every other changed file, so cover as many parts as a diff needs only up +# An entry costs about a minute and 2 GB, against a 15 minute job budget shared +# with every other changed file, so keep as many entries as a diff needs only up # to this many. Beyond it the summary line names the parts left unchecked # rather than letting the job run into its timeout, which is the silent failure # this whole reduction exists to remove. @@ -75,10 +76,13 @@ return list(entry.get("arguments", ())) -def part_of(entry): - """The ``EIGEN_TEST_PART`` an entry defines, as a string, or None.""" - found = _PART_ARG.search(entry.get("command") or " ".join(entry.get("arguments", ()))) - return found.group(1) if found else None +def parts_of(entry): + """The ``EIGEN_TEST_PART``s an entry defines, as a frozenset of strings. + + An entry for a range in ``cmake/EigenTestPartGroups.cmake`` defines every + part of the range; any other entry of a split test defines one. + """ + return frozenset(_PART_ARG.findall(entry.get("command") or " ".join(entry.get("arguments", ())))) def configuration_of(entry): @@ -224,23 +228,21 @@ for number in lines if number < len(guards)] -def select_parts(required, order, limit=MAX_PARTS): - """The smallest set of parts covering ``required``, capped at ``limit``. +def select_entries(required, entries, limit=MAX_PARTS): + """The fewest entries whose parts cover ``required``, capped at ``limit``. - Returns the chosen parts and the requirements the cap left uncovered. Ties - go to the part the database lists first, which is what a diff that names no - part at all gets. + ``entries`` holds the parts each entry defines, in database order. Returns + the positions chosen in ``entries`` and the requirements the cap left + uncovered. Ties go to the entry the database lists first, which is what a + diff that names no part at all gets. """ chosen = [] uncovered = [need for need in required if need] while uncovered and len(chosen) < limit: - counts = {} - for need in uncovered: - for part in need: - counts[part] = counts.get(part, 0) + 1 - best = min(counts, key=lambda part: (-counts[part], order.index(part))) + counts = [sum(1 for need in uncovered if need & members) for members in entries] + best = max(range(len(entries)), key=lambda position: (counts[position], -position)) chosen.append(best) - uncovered = [need for need in uncovered if best not in need] + uncovered = [need for need in uncovered if not need & entries[best]] return chosen, uncovered @@ -256,18 +258,18 @@ keep, skipped, unreachable = set(), set(), 0 for group in groups.values(): - parts = [part_of(matches[index]) for index in group] - if len(group) == 1 or None in parts or len(set(parts)) != len(parts): - # Not one configuration split into parts: every entry here parses - # something the others do not. + parts = [parts_of(matches[index]) for index in group] + universe = frozenset().union(*parts) + if len(group) == 1 or not all(parts) or sum(map(len, parts)) != len(universe): + # Not one configuration split into disjoint parts: every entry here + # parses something the others do not. keep.update(group) continue - universe = frozenset(parts) required = required_parts(text, lines, universe) unreachable = max(unreachable, sum(1 for need in required if not need)) - chosen, uncovered = select_parts(required, parts) + chosen, uncovered = select_entries(required, parts) skipped |= {part for need in uncovered for part in need} - keep.update(index for index, part in zip(group, parts) if part in chosen) + keep.update(group[position] for position in chosen) return [matches[index] for index in sorted(keep)], sorted(skipped, key=int), unreachable @@ -288,7 +290,7 @@ """ if len(keep) == len(matches) and not skipped and not unreachable: return "" - parts = sorted((part for part in (part_of(entry) for entry in keep) if part), key=int) + parts = sorted({part for entry in keep for part in parts_of(entry)}, key=int) if not parts: checking = "%d entries" % len(keep) else:
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dc54759..004407f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt
@@ -318,6 +318,9 @@ ei_add_test(packet_segment) ei_add_test(vectorization_logic) ei_add_test(basicstuff) +ei_add_test(approx_comparisons) +ei_add_test(numerical_test_helpers) +ei_add_test(fuzzy_matrix_checks) ei_add_test(constexpr) ei_add_test(constructor) function(eigen_check_std_variant) @@ -354,15 +357,25 @@ ei_add_test(redux) ei_add_test(redux_bounded) if(CMAKE_COMPILER_IS_GNUCXX) - # Keep this diagnostic isolated from the runtime suite's warning policy and packet reductions. + # Check optimized bounds diagnostics: scalar with assertions, vectorized without. add_executable(redux_bounded_compile EXCLUDE_FROM_ALL redux_bounded_compile.cpp) target_link_libraries(redux_bounded_compile PRIVATE Eigen3::Eigen) - target_compile_options(redux_bounded_compile PRIVATE -O3 -Werror=array-bounds) + target_compile_options(redux_bounded_compile PRIVATE -O3 -Werror=array-bounds -UNDEBUG) target_compile_definitions(redux_bounded_compile PRIVATE EIGEN_DONT_VECTORIZE) add_dependencies(redux_bounded redux_bounded_compile) + add_executable(redux_bounded_compile_vectorized EXCLUDE_FROM_ALL redux_bounded_compile_vectorized.cpp) + # EIGEN_TEST_CUSTOM_LINKER_FLAGS accompanies the custom compile flags below: sanitizer flags need both. + target_link_libraries(redux_bounded_compile_vectorized PRIVATE Eigen3::Eigen ${EIGEN_TEST_CUSTOM_LINKER_FLAGS}) + # The job's ISA flags, as for bug1213 below: without them an SME build defines EIGEN_ARM64_USE_SME with no SME + # target and stops at an #error, and other jobs check a packet path they did not select. They go first so that -O3 + # overrides an optimization level among them. + target_compile_options(redux_bounded_compile_vectorized PRIVATE ${EIGEN_TEST_CUSTOM_CXX_FLAGS} -O3 -Werror=array-bounds) + target_compile_definitions(redux_bounded_compile_vectorized PRIVATE EIGEN_NO_DEBUG) + add_dependencies(redux_bounded redux_bounded_compile_vectorized) endif() ei_add_test(visitor) ei_add_test(block) +ei_add_test(block_coeffwise) ei_add_test(corners) ei_add_test(symbolic_index) ei_add_test(indexed_view) @@ -373,11 +386,13 @@ ei_add_test(conservative_resize) ei_add_test(product_small) ei_add_test(product_large) +ei_add_test(product_threshold) if(EIGEN_TEST_SME) # EIGEN_TEST_SME defines EIGEN_ARM64_USE_SME (root CMakeLists.txt), which turns # a toolchain without SME2 into a build error rather than a NEON fallback; the # toolchain must still supply an SME2 -march/-mcpu. ei_add_test(product_sme) + ei_add_test(vector_sme "-Werror=unknown-pragmas") endif() if(EIGEN_TEST_SVE) # EIGEN_TEST_SVE defines EIGEN_ARM64_USE_SVE and -msve-vector-bits (root @@ -402,6 +417,7 @@ ei_add_test(unaryview) ei_add_test(mapstaticmethods) ei_add_test(array_cwise) +ei_add_test(unary_pow) ei_add_test(matrix_cwise) ei_add_test(array_for_matrix) ei_add_test(array_replicate) @@ -411,6 +427,7 @@ ei_add_test(is_same_dense) ei_add_test(triangular) ei_add_test(trsm_packet) +ei_add_test(trsm_packet_row_major_default) ei_add_test(trsm_packet_cancellation) ei_add_test(triangular_in_place) ei_add_test(selfadjoint) @@ -546,9 +563,9 @@ if(COMPILER_SUPPORT_FASTMATH) set(EIGEN_FASTMATH_FLAGS "-ffast-math") else() - check_cxx_compiler_flag("/fp:fast" COMPILER_SUPPORT_FPFAST) + check_cxx_compiler_flag("-fp:fast" COMPILER_SUPPORT_FPFAST) if(COMPILER_SUPPORT_FPFAST) - set(EIGEN_FASTMATH_FLAGS "/fp:fast") + set(EIGEN_FASTMATH_FLAGS "-fp:fast") endif() endif() # The fastmath test intentionally uses NaN/infinity under -ffast-math.
diff --git a/test/CustomComplex.h b/test/CustomComplex.h index 079d417..e1cdfb3 100644 --- a/test/CustomComplex.h +++ b/test/CustomComplex.h
@@ -130,4 +130,13 @@ } // namespace numext } // namespace Eigen +namespace custom_complex { +// Scalar comparison for the test macros; the coefficient-wise helpers reach it through ADL. +template <typename Real> +bool test_isApprox(const CustomComplex<Real>& a, const CustomComplex<Real>& b) { + using Vector = Eigen::Matrix<CustomComplex<Real>, 1, 1>; + return Vector::Constant(a).isApprox(Vector::Constant(b), Eigen::test_precision<Real>()); +} +} // namespace custom_complex + #endif // EIGEN_TEST_CUSTOM_COMPLEX_H
diff --git a/test/adjoint.cpp b/test/adjoint.cpp index 88d52d0..bf1fb57 100644 --- a/test/adjoint.cpp +++ b/test/adjoint.cpp
@@ -10,6 +10,13 @@ #include "main.h" +// |value|, promoted so that the error bounds below accumulate in the widest available type. +template <typename T> +long double wide_abs(const T& value) { + using std::abs; + return static_cast<long double>(abs(value)); +} + template <bool IsInteger> struct adjoint_specific; @@ -17,12 +24,11 @@ struct adjoint_specific<true> { template <typename Vec, typename Mat, typename Scalar> static void run(const Vec& v1, const Vec& v2, Vec& v3, const Mat& square, Scalar s1, Scalar s2) { - VERIFY(test_isApproxWithRef((s1 * v1 + s2 * v2).dot(v3), - numext::conj(s1) * v1.dot(v3) + numext::conj(s2) * v2.dot(v3), 0)); - VERIFY(test_isApproxWithRef(v3.dot(s1 * v1 + s2 * v2), s1 * v3.dot(v1) + s2 * v3.dot(v2), 0)); + VERIFY_IS_EQUAL((s1 * v1 + s2 * v2).dot(v3), numext::conj(s1) * v1.dot(v3) + numext::conj(s2) * v2.dot(v3)); + VERIFY_IS_EQUAL(v3.dot(s1 * v1 + s2 * v2), s1 * v3.dot(v1) + s2 * v3.dot(v2)); // check compatibility of dot and adjoint - VERIFY(test_isApproxWithRef(v1.dot(square * v2), (square.adjoint() * v1).dot(v2), 0)); + VERIFY_IS_EQUAL(v1.dot(square * v2), (square.adjoint() * v1).dot(v2)); } }; @@ -30,13 +36,19 @@ struct adjoint_specific<false> { template <typename Vec, typename Mat, typename Scalar> static void run(const Vec& v1, const Vec& v2, Vec& v3, const Mat& square, Scalar s1, Scalar s2) { - typedef typename NumTraits<Scalar>::Real RealScalar; - using std::abs; + using RealScalar = typename NumTraits<Scalar>::Real; - RealScalar ref = NumTraits<Scalar>::IsInteger ? RealScalar(0) : (std::max)((s1 * v1 + s2 * v2).norm(), v3.norm()); - VERIFY(test_isApproxWithRef((s1 * v1 + s2 * v2).dot(v3), - numext::conj(s1) * v1.dot(v3) + numext::conj(s2) * v2.dot(v3), ref)); - VERIFY(test_isApproxWithRef(v3.dot(s1 * v1 + s2 * v2), s1 * v3.dot(v1) + s2 * v3.dot(v2), ref)); + const long double eps = static_cast<long double>(NumTraits<RealScalar>::epsilon()); + const long double absS1 = wide_abs(s1), absS2 = wide_abs(s2); + long double linearityScale = 0; + for (Index i = 0; i < v1.size(); ++i) + linearityScale += (absS1 * wide_abs(v1(i)) + absS2 * wide_abs(v2(i))) * wide_abs(v3(i)); + // Two evaluation orders, including complex multiply-adds and scalar-vector products. + const long double linearityBound = 8 * (v1.size() + 4) * eps * linearityScale; + VERIFY((numext::isfinite)(linearityBound)); + VERIFY(wide_abs((s1 * v1 + s2 * v2).dot(v3) - (numext::conj(s1) * v1.dot(v3) + numext::conj(s2) * v2.dot(v3))) <= + linearityBound); + VERIFY(wide_abs(v3.dot(s1 * v1 + s2 * v2) - (s1 * v3.dot(v1) + s2 * v3.dot(v2))) <= linearityBound); VERIFY_IS_APPROX(v1.squaredNorm(), v1.norm() * v1.norm()); // check normalized() and normalize() @@ -51,19 +63,25 @@ VERIFY_IS_APPROX((v1 * 0).normalized(), (v1 * 0)); #if (!EIGEN_ARCH_i386) || defined(EIGEN_VECTORIZE) RealScalar very_small = (std::numeric_limits<RealScalar>::min)(); - VERIFY(numext::is_exactly_zero((v1 * very_small).norm())); - VERIFY_IS_APPROX((v1 * very_small).normalized(), (v1 * very_small)); - v3 = v1 * very_small; + // Materialized once: ARMv7 NEON flushes the subnormal products that a coefficient-wise read of the lazy + // expression keeps. + const Vec tiny = v1 * very_small; + VERIFY(numext::is_exactly_zero(tiny.norm())); + VERIFY_IS_APPROX(tiny.normalized(), tiny); + v3 = tiny; v3.normalize(); - VERIFY_IS_APPROX(v3, (v1 * very_small)); + VERIFY_IS_APPROX(v3, tiny); #endif // check compatibility of dot and adjoint - ref = NumTraits<Scalar>::IsInteger ? 0 - : (std::max)((std::max)(v1.norm(), v2.norm()), - (std::max)((square * v2).norm(), (square.adjoint() * v1).norm())); - VERIFY(internal::isMuchSmallerThan(abs(v1.dot(square * v2) - (square.adjoint() * v1).dot(v2)), ref, - test_precision<Scalar>())); + long double adjointScale = 0; + for (Index i = 0; i < square.rows(); ++i) + for (Index j = 0; j < square.cols(); ++j) + adjointScale += wide_abs(v1(i)) * wide_abs(square(i, j)) * wide_abs(v2(j)); + // Each order has two reductions; allow 8*n*eps per complex product and its accumulation. + const long double adjointBound = 16 * (v1.size() + 1) * eps * adjointScale; + VERIFY((numext::isfinite)(adjointBound)); + VERIFY(wide_abs(v1.dot(square * v2) - (square.adjoint() * v1).dot(v2)) <= adjointBound); // check that Random().normalized() works: tricky as the random xpr must be evaluated by // normalized() in order to produce a consistent result.
diff --git a/test/approx_comparisons.cpp b/test/approx_comparisons.cpp new file mode 100644 index 0000000..3f28a1b --- /dev/null +++ b/test/approx_comparisons.cpp
@@ -0,0 +1,355 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#define EIGEN_RUNTIME_NO_MALLOC +#include "main.h" +#include <Eigen/Core> +#include <Eigen/Geometry> +#include "fp_control.h" +#define EIGEN_TEST_ANNOYING_SCALAR_DONT_THROW +#include "AnnoyingScalar.h" +#include "CustomComplex.h" + +template <typename Scalar, int Options> +void approx_comparisons_floating() { + using Real = typename NumTraits<Scalar>::Real; + using MatrixType = Matrix<Scalar, Dynamic, Dynamic, Options>; + const Real precision = Real(0.125); + MatrixType x(3, 5), y(3, 5); + for (const Real scale : + {Real(1), NumTraits<Real>::highest() / Real(32), (numext::numeric_limits<Real>::min)() * Real(32)}) { + x.setConstant(Scalar(scale)); + y = x * Real(2); + internal::set_is_malloc_allowed(false); + VERIFY(x.isApprox(x, precision)); + VERIFY(!x.isApprox(y, precision)); + VERIFY(!y.isApprox(x, precision)); + VERIFY(!x.array().isApprox(y.array(), precision)); + VERIFY(!x.transpose().isApprox(y.transpose(), precision)); + VERIFY(!x.isMuchSmallerThan(y, precision)); + VERIFY(x.isMuchSmallerThan(y, Real(0.5))); + VERIFY(!x.isMuchSmallerThan(scale, precision)); + VERIFY(x.isMuchSmallerThan(Real(4) * scale, Real(1))); + VERIFY(x.isApprox(y, Real(1))); + VERIFY(x.isApprox(y, Real(-1))); + VERIFY(x.isApprox(x, Real(0))); + VERIFY(!x.isApprox(y, Real(0))); + y = x * (Real(1) + precision / Real(2)); + VERIFY(x.isApprox(y, precision)); + VERIFY(x.array().isApprox(y.array(), precision)); + VERIFY((x.template block<2, 3>(0, 1).isApprox(y.template block<2, 3>(0, 1), precision))); + internal::set_is_malloc_allowed(true); + // The scaled path evaluates a lazy operand, which allocates for dynamic sizes. + VERIFY(!x.isApprox(MatrixType::Zero(3, 5), precision)); + Matrix<Scalar, Dynamic, Dynamic, Options == ColMajor ? RowMajor : ColMajor> other = y; + VERIFY(x.isApprox(other, precision)); + } + + // Norms and differences can exceed the scalar range even though every coefficient is finite. + x.setConstant(Scalar(NumTraits<Real>::highest())); + y = -x; + VERIFY(x.isApprox(x, Real(0))); + VERIFY(!x.isApprox(y, Real(1))); + VERIFY(x.isApprox(y, Real(2))); + VERIFY(!x.isMuchSmallerThan(x, precision)); + // The squared difference, but not the bound, can overflow the accumulator here. + const Real root = numext::ldexp(Real(1), (NumTraits<Real>::max_exponent() - 1) / 2); + x.setZero(); + x(0, 0) = Scalar(root); + y = -x; + VERIFY(!x.isApprox(y, Real(1.9))); + VERIFY(x.isApprox(y, Real(2))); + + // Squaring either the difference or the tolerance would underflow. + const Real tiny = (numext::numeric_limits<Real>::min)() * Real(16); + x.setZero(); + y.setZero(); + x(0, 0) = y(0, 0) = Scalar(1); + x(2, 4) = Scalar(tiny); + VERIFY(!x.isApprox(y, Real(0))); + VERIFY(!x.isApprox(y, tiny / Real(2))); + VERIFY(x.isApprox(y, tiny * Real(2))); + VERIFY(x.isApprox(y, NumTraits<Real>::highest())); + x.setZero(); + VERIFY(x.isMuchSmallerThan(y, Real(0))); + VERIFY(x.isMuchSmallerThan(Real(1), Real(0))); + + for (Index rows : {Index(0), Index(3)}) { + x.resize(rows, 0); + y.resize(rows, 0); + VERIFY(x.isApprox(y)); + VERIFY(x.isMuchSmallerThan(y)); + VERIFY(x.isMuchSmallerThan(Real(0))); + } +} + +// The scaled path views plain objects, maps and their blocks in place, whatever their strides, and evaluates other +// operands, on the stack when their size is bounded. +template <typename Scalar> +void approx_comparisons_strided_operands() { + using Real = typename NumTraits<Scalar>::Real; + using ColMatrix = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; + using RowMatrix = Matrix<Scalar, Dynamic, Dynamic, RowMajor>; + using StridedMap = Map<const ColMatrix, 0, Stride<Dynamic, Dynamic>>; + const Real precision = Real(0.125); + // Squares of these coefficients underflow, which selects the scaled path. + const Real scale = (numext::numeric_limits<Real>::min)() * Real(32); + const ColMatrix x = ColMatrix::Random(6, 8) * scale; + const ColMatrix y = x * (Real(1) + precision / Real(2)); + const ColMatrix z = x * Real(2); + const RowMatrix xr = x, yr = y, zr = z; + const Stride<Dynamic, Dynamic> stride(12, 2); + const ColMatrix ys = StridedMap(y.data(), 3, 4, stride); + internal::set_is_malloc_allowed(false); + VERIFY(xr.isApprox(y, precision)); + VERIFY(!xr.isApprox(z, precision)); + VERIFY(x.row(2).isApprox(yr.row(2), precision)); + VERIFY(!x.row(2).isApprox(zr.row(2), precision)); + VERIFY(xr.col(3).isApprox(y.col(3), precision)); + VERIFY(!xr.col(3).isApprox(z.col(3), precision)); + VERIFY(StridedMap(x.data(), 3, 4, stride).isApprox(ys, precision)); + VERIFY(!StridedMap(x.data(), 3, 4, stride).isApprox(StridedMap(z.data(), 3, 4, stride), precision)); + VERIFY((Real(2) * x.template topLeftCorner<3, 3>()).isApprox(Real(2) * y.template topLeftCorner<3, 3>(), precision)); + VERIFY(!(Real(2) * x.template topLeftCorner<3, 3>()).isApprox(x.template topLeftCorner<3, 3>(), precision)); + internal::set_is_malloc_allowed(true); + // These have direct access, but their strides do not address every coefficient. + VERIFY(x(2, seq(0, 7, 2)).isApprox(yr(2, seq(0, 7, 2)), precision)); + VERIFY(xr(seq(0, 5, 2), 3).isApprox(y(seq(0, 5, 2), 3), precision)); + VERIFY(x.row(2).realView().isApprox(yr.row(2).realView(), precision)); +} + +template <typename Real> +void approx_comparisons_rounding_boundary() { + using Vector = Matrix<Real, 1, 1>; + const Real eps = NumTraits<Real>::epsilon(); + // One nonzero component and power-of-two scales make the relative difference exactly 8*eps. + for (int exponent : {0, NumTraits<Real>::min_exponent() + 4, NumTraits<Real>::max_exponent() - 8}) { + const Real scale = numext::ldexp(Real(1), exponent); + const Vector x = Vector::Constant(scale); + const Vector y = Vector::Constant(scale * (Real(1) + Real(8) * eps)); + VERIFY(!x.isApprox(y, Real(7) * eps)); + VERIFY(x.isApprox(y, Real(8) * eps)); + VERIFY(x.isApprox(y, Real(9) * eps)); + } +} + +template <typename Real> +void approx_comparisons_flush_to_zero() { + ScopedFlushToZero flushToZero; + using Vector = Matrix<Real, 2, 1>; + // The difference of these normal operands is subnormal. + const Real normalMin = (numext::numeric_limits<Real>::min)(); + const Vector x(Real(1.5) * normalMin, Real(0)), y(normalMin, Real(0)); + VERIFY(!x.isApprox(y, Real(0.25))); + VERIFY(!x.isApprox(y, Real(0))); + VERIFY(x.isApprox(y, Real(0.5))); + VERIFY_IS_EQUAL(test_relative_error(x, y), Real(0.5)); + approx_comparisons_rounding_boundary<Real>(); +} + +// The squared differences, 2^-118 and just below 2^-126, accumulate in float, where FTZ flushes the second. +template <typename = void> +void approx_comparisons_flush_to_zero_bfloat16() { + ScopedFlushToZero flushToZero; + using Vector = Matrix<bfloat16, 2, 1>; + // Opaque, so the compiler cannot square it with IEEE subnormals at compile time. + volatile float tail = numext::ldexp(1.9921875f, -64); + const Vector x(bfloat16(numext::ldexp(1.0f, -52)), bfloat16(0.0f)); + const Vector y(bfloat16(numext::ldexp(1.0f + 1.0f / 128, -52)), bfloat16(float(tail))); + VERIFY(!x.isApprox(y, bfloat16(1.0f / 128))); +} + +template <typename Real> +void approx_comparisons_special_values() { + using Vector = Matrix<Real, 2, 1>; + const Real inf = NumTraits<Real>::infinity(); + const Real nan = NumTraits<Real>::quiet_NaN(); + Vector x = Vector::Ones(), y = Vector::Ones(); + for (Index i = 0; i < 2; ++i) { + y = x; + y(i) = nan; + VERIFY(!x.isApprox(y)); + VERIFY(!y.isApprox(x)); + VERIFY(!y.isApprox(y)); + VERIFY(!x.isMuchSmallerThan(y)); + VERIFY(!y.isMuchSmallerThan(x)); + y(i) = inf; + VERIFY(!x.isApprox(y)); + VERIFY(!y.isApprox(y)); + VERIFY(!y.isMuchSmallerThan(y)); + VERIFY(x.isMuchSmallerThan(y)); + } + VERIFY(!x.isApprox(x, nan)); + VERIFY(!x.isMuchSmallerThan(x, nan)); + VERIFY(!x.isMuchSmallerThan(nan)); + VERIFY(!x.isMuchSmallerThan(inf, Real(0))); + + if (subnormalDivisionIsExact<Real>()) { + const Real tiny = (numext::numeric_limits<Real>::denorm_min)(); + x.setConstant(tiny); + y = Real(2) * x; + VERIFY(!x.isApprox(y)); + VERIFY(x.isApprox(x, Real(0))); + VERIFY(!x.isApprox(Vector::Zero())); + VERIFY(!x.isMuchSmallerThan(y)); + VERIFY(!x.isMuchSmallerThan(tiny)); + } +} + +template <typename Scalar> +void approx_comparisons_exact() { + using MatrixType = Matrix<Scalar, 2, 2>; + const MatrixType zero = MatrixType::Zero(); + const MatrixType one = MatrixType::Ones(); + VERIFY(zero.isApprox(zero)); + VERIFY(one.isApprox(one)); + VERIFY(!zero.isApprox(one)); + VERIFY(!one.isApprox(zero)); + VERIFY(zero.isMuchSmallerThan(one)); + VERIFY(zero.isMuchSmallerThan(Scalar(0))); + VERIFY(!one.isMuchSmallerThan(one)); + VERIFY(!one.isMuchSmallerThan(Scalar(1))); +} + +template <typename = void> +void approx_comparisons_expressions() { + Matrix2d x; + x << 1, 2, 3, 4; + VERIFY((x * x).isApprox((x * x).eval())); + VERIFY((x + x).isApprox(2 * x)); + VERIFY((x * x * 0).isMuchSmallerThan(x * x)); + VERIFY((x * x * 0).isMuchSmallerThan(1.0)); + VERIFY(!Matrix2f::Ones().isMuchSmallerThan(Matrix2d::Constant(1e200), 0.0f)); + VERIFY(Matrix2f::Ones().isMuchSmallerThan(Matrix2d::Constant(1e200))); + VERIFY(!Matrix2d::Ones().isMuchSmallerThan(Matrix2f::Constant(1e-30f))); + VERIFY(Matrix2d::Constant(1e-300).isMuchSmallerThan(Matrix2f::Constant(1e10f), 1e-300)); + VERIFY(!Matrix2f::Constant(1e-23f).isMuchSmallerThan(Matrix2d::Constant(1e-20), 1e-5f)); + VERIFY(Matrix2f::Constant(1e20f).isMuchSmallerThan(Matrix2d::Constant(1e30), 1e-5f)); + const Matrix2d tiny = Matrix2d::Constant(1e-200); + const Matrix2cd complexTiny = tiny.cast<std::complex<double>>(); + VERIFY(tiny.isApprox(complexTiny)); + VERIFY(complexTiny.isApprox(tiny)); + STATIC_CHECK(!internal::use_scaled_comparison<AnnoyingScalar>::value); + approx_comparisons_exact<AnnoyingScalar>(); + const Matrix<AnnoyingScalar, 2, 2> custom = Matrix<AnnoyingScalar, 2, 2>::Ones(); + VERIFY(test_isCwiseApprox(custom, custom, false)); + VERIFY(test_relative_error(custom, custom) == AnnoyingScalar(0)); + // A custom complex scalar over a built-in real has no dedicated coefficient overload. + using CustomComplexScalar = CustomComplex<double>; + STATIC_CHECK(!internal::use_scaled_comparison<CustomComplexScalar>::value); + const Matrix<CustomComplexScalar, 2, 2> customComplex = + Matrix<CustomComplexScalar, 2, 2>::Constant(CustomComplexScalar(1, 2)); + VERIFY(test_isCwiseApprox(customComplex, customComplex, true)); + VERIFY(test_isCwiseApprox(customComplex, customComplex, false)); + VERIFY(!test_isCwiseApprox(customComplex, (customComplex * CustomComplexScalar(2)).eval(), false)); + VERIFY_IS_EQUAL(test_relative_error(customComplex, customComplex), 0.0); +} + +template <typename Real> +void approx_comparisons_complex_components() { + using Complex = std::complex<Real>; + using Vector = Matrix<Complex, 2, 1>; + const Real big = NumTraits<Real>::highest(); + const Vector x = Vector::Constant(Complex(big, big)); + const Vector y = -x; + VERIFY(x.isApprox(x, Real(0))); + VERIFY(!x.isApprox(y, Real(1))); + VERIFY(x.isApprox(y, Real(2))); + VERIFY(!x.isMuchSmallerThan(y, Real(0.5))); + VERIFY(!x.isMuchSmallerThan(big, Real(1))); +} + +// nested_eval keeps these cheap expressions lazy, so they reach the scaled path without direct access. +template <typename Real> +void approx_comparisons_lazy_operands() { + using Complex = std::complex<Real>; + using Vector3 = Matrix<Real, 3, 1>; + using Vector4 = Matrix<Real, 4, 1>; + using Array4 = Array<Real, 4, 1>; + const Real precision = Real(0.125); + const Array4 base = Array4::LinSpaced(Real(1), Real(2)); + for (int exponent : {0, NumTraits<Real>::min_exponent() + 4, NumTraits<Real>::max_exponent() - 8}) { + const Real scale = numext::ldexp(Real(1), exponent); + const Vector3 x = Vector3::Constant(scale); + Vector4 y; + y << x, Real(1); + VERIFY(x.homogeneous().isApprox(y, precision)); + VERIFY(y.isApprox(x.homogeneous(), precision)); + VERIFY_IS_EQUAL(test_relative_error(x.homogeneous(), y), Real(0)); + VERIFY(!x.homogeneous().isMuchSmallerThan(y, precision)); + y *= Real(2); + VERIFY(!x.homogeneous().isApprox(y, precision)); + VERIFY(!y.isApprox(x.homogeneous(), precision)); + VERIFY(test_relative_error(x.homogeneous(), y) > precision); + + const Array4 a = base * scale; + VERIFY((a + a).isApprox(a * Real(2), precision)); + VERIFY(!(a + a).isApprox(a, precision)); + VERIFY_IS_EQUAL(test_relative_error(a + a, a * Real(2)), Real(0)); + } + // A unary functor whose result is complex but whose argument is real. + const Complex power(Real(2), Real(1)); + VERIFY(base.pow(power).isApprox(base.template cast<Complex>().pow(power), precision)); + VERIFY(test_relative_error(base.pow(power), base.template cast<Complex>().pow(power)) <= precision); +} + +template <typename Real> +void approx_comparisons_scale_invariance() { + using Vector = Matrix<Real, 17, 1>; + const Real precision = Real(0.125); + const long double precision2 = static_cast<long double>(precision) * static_cast<long double>(precision); + for (int repeat = 0; repeat < g_repeat; ++repeat) { + const Vector a = Vector::Random(); + for (int close = 0; close < 2; ++close) { + Vector b = Vector::Random(); + if (close) b = a + precision * Real(0.25) * b; + long double difference = 0, a2 = 0, b2 = 0; + for (Index i = 0; i < a.size(); ++i) { + const long double av = static_cast<long double>(a(i)); + const long double bv = static_cast<long double>(b(i)); + difference += (av - bv) * (av - bv); + a2 += av * av; + b2 += bv * bv; + } + const bool expected = difference <= precision2 * (std::min)(a2, b2); + for (int exponent : {0, NumTraits<Real>::min_exponent() + 4, NumTraits<Real>::max_exponent() - 8}) { + const Real scale = numext::ldexp(Real(1), exponent); + const Vector x = a * scale, y = b * scale; + VERIFY_IS_EQUAL(x.isApprox(y, precision), expected); + } + } + } +} + +EIGEN_DECLARE_TEST(approx_comparisons) { + CALL_SUBTEST_1((approx_comparisons_floating<float, ColMajor>())); + CALL_SUBTEST_1((approx_comparisons_floating<double, RowMajor>())); + CALL_SUBTEST_1(approx_comparisons_strided_operands<float>()); + CALL_SUBTEST_1(approx_comparisons_strided_operands<double>()); + CALL_SUBTEST_1(approx_comparisons_special_values<float>()); + CALL_SUBTEST_1(approx_comparisons_special_values<double>()); + CALL_SUBTEST_1(approx_comparisons_rounding_boundary<float>()); + CALL_SUBTEST_1(approx_comparisons_rounding_boundary<double>()); + CALL_SUBTEST_1(approx_comparisons_scale_invariance<float>()); + CALL_SUBTEST_1(approx_comparisons_scale_invariance<double>()); + CALL_SUBTEST_1(approx_comparisons_flush_to_zero<float>()); + CALL_SUBTEST_1(approx_comparisons_flush_to_zero<double>()); + CALL_SUBTEST_2((approx_comparisons_floating<std::complex<float>, RowMajor>())); + CALL_SUBTEST_2((approx_comparisons_floating<std::complex<double>, ColMajor>())); + CALL_SUBTEST_2(approx_comparisons_strided_operands<std::complex<float>>()); + CALL_SUBTEST_2(approx_comparisons_complex_components<float>()); + CALL_SUBTEST_2(approx_comparisons_complex_components<double>()); + CALL_SUBTEST_2(approx_comparisons_lazy_operands<float>()); + CALL_SUBTEST_2(approx_comparisons_lazy_operands<double>()); + CALL_SUBTEST_3((approx_comparisons_floating<half, ColMajor>())); + CALL_SUBTEST_3((approx_comparisons_floating<bfloat16, RowMajor>())); + CALL_SUBTEST_3((approx_comparisons_floating<long double, ColMajor>())); + CALL_SUBTEST_3(approx_comparisons_rounding_boundary<half>()); + CALL_SUBTEST_3(approx_comparisons_rounding_boundary<bfloat16>()); + CALL_SUBTEST_3(approx_comparisons_flush_to_zero<bfloat16>()); + CALL_SUBTEST_3(approx_comparisons_flush_to_zero_bfloat16<>()); + CALL_SUBTEST_4(approx_comparisons_exact<int>()); + CALL_SUBTEST_4(approx_comparisons_exact<unsigned int>()); + CALL_SUBTEST_4(approx_comparisons_exact<bool>()); + CALL_SUBTEST_4(approx_comparisons_expressions<>()); +}
diff --git a/test/array_cwise.cpp b/test/array_cwise.cpp index 0cbdfe5..07de657 100644 --- a/test/array_cwise.cpp +++ b/test/array_cwise.cpp
@@ -468,11 +468,13 @@ } } +template <typename = void> void float_pow_test() { float_pow_test_impl<float, float>(); float_pow_test_impl<double, double>(); } +template <typename = void> void mixed_pow_test() { // The following cases will test promoting a smaller exponent type // to a wider base type. @@ -495,6 +497,7 @@ // unary_pow_test<float, double>(); } +template <typename = void> void int_pow_test() { int_pow_test_impl<int, int>(); int_pow_test_impl<unsigned int, unsigned int>(); @@ -550,6 +553,7 @@ VERIFY(all_pass); } +template <typename = void> void signbit_tests() { signbit_test<float>(); signbit_test<double>(); @@ -1416,6 +1420,7 @@ uint32_t, uint64_t, float, double, /*long double, */ half, bfloat16>::run(); } +template <typename = void> void bool_logical_ops() { const Index size = 67; ArrayX<bool> lhs = ArrayXi::Random(size) > 0; @@ -1560,10 +1565,10 @@ CALL_SUBTEST_18(complex_classification<long double>()); for (int i = 0; i < g_repeat; i++) { - CALL_SUBTEST_19(float_pow_test()); - CALL_SUBTEST_20(int_pow_test()); - CALL_SUBTEST_21(mixed_pow_test()); - CALL_SUBTEST_22(signbit_tests()); + CALL_SUBTEST_19(float_pow_test<>()); + CALL_SUBTEST_20(int_pow_test<>()); + CALL_SUBTEST_21(mixed_pow_test<>()); + CALL_SUBTEST_22(signbit_tests<>()); } for (int i = 0; i < g_repeat; i++) { CALL_SUBTEST_34(ldexp_test<float>()); @@ -1581,7 +1586,7 @@ CALL_SUBTEST_25(typed_logicals_test(ArrayX<double>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); CALL_SUBTEST_26(typed_logicals_test(ArrayX<std::complex<float>>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); CALL_SUBTEST_27(typed_logicals_test(ArrayX<std::complex<double>>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); - CALL_SUBTEST_42(bool_logical_ops()); + CALL_SUBTEST_42(bool_logical_ops<>()); } for (int i = 0; i < g_repeat; i++) {
diff --git a/test/array_for_matrix.cpp b/test/array_for_matrix.cpp index 6ed31ef..4c8e7d8 100644 --- a/test/array_for_matrix.cpp +++ b/test/array_for_matrix.cpp
@@ -186,6 +186,10 @@ // min/max with array Scalar maxM1 = m1.maxCoeff(); Scalar minM1 = m1.minCoeff(); + // Reduce here: GCC 10 for AArch64 can keep an unreduced 128-bit partial result in v8 across the calls below, + // although AAPCS64 preserves only its low 64 bits. + EIGEN_OPTIMIZATION_BARRIER(maxM1); + EIGEN_OPTIMIZATION_BARRIER(minM1); VERIFY_IS_APPROX(MatrixType::Constant(rows, cols, minM1), m1.cwiseMin(MatrixType::Constant(rows, cols, minM1))); VERIFY_IS_APPROX(m1, m1.cwiseMin(MatrixType::Constant(rows, cols, maxM1)));
diff --git a/test/array_replicate.cpp b/test/array_replicate.cpp index 0e7587c..305f14d 100644 --- a/test/array_replicate.cpp +++ b/test/array_replicate.cpp
@@ -62,6 +62,85 @@ VERIFY_IS_APPROX(vx1, v1.colwise().replicate(f2)); } +template <typename ExpressionType> +void vectorwise_replicate_indexers(ExpressionType& m) { + using Scalar = typename ExpressionType::Scalar; + Matrix<Scalar, Dynamic, Dynamic> vertical(3 * m.rows(), m.cols()); + Matrix<Scalar, Dynamic, Dynamic> horizontal(m.rows(), 3 * m.cols()); + for (Index j = 0; j < vertical.cols(); ++j) + for (Index i = 0; i < vertical.rows(); ++i) vertical(i, j) = m(i % m.rows(), j); + for (Index j = 0; j < horizontal.cols(); ++j) + for (Index i = 0; i < horizontal.rows(); ++i) horizontal(i, j) = m(i, j % m.cols()); + + VERIFY_IS_EQUAL(m.colwise().replicate(fix<3>).matrix(), vertical); + VERIFY_IS_EQUAL(m.colwise().replicate(fix<3>(3)).matrix(), vertical); + VERIFY_IS_EQUAL(m.colwise().replicate(fix<Dynamic>(2)).matrix(), vertical.topRows(2 * m.rows())); + VERIFY_IS_EQUAL(m.colwise().replicate(3).matrix(), vertical); + VERIFY_IS_EQUAL(m.colwise().replicate(Index(3)).matrix(), vertical); + VERIFY_IS_EQUAL(m.colwise().replicate(fix<1>).matrix(), m.matrix()); + VERIFY_IS_EQUAL(m.colwise().replicate(fix<0>).rows(), 0); + VERIFY_IS_EQUAL(m.rowwise().replicate(fix<3>).matrix(), horizontal); + VERIFY_IS_EQUAL(m.rowwise().replicate(fix<3>(3)).matrix(), horizontal); + VERIFY_IS_EQUAL(m.rowwise().replicate(fix<Dynamic>(2)).matrix(), horizontal.leftCols(2 * m.cols())); + VERIFY_IS_EQUAL(m.rowwise().replicate(3).matrix(), horizontal); + VERIFY_IS_EQUAL(m.rowwise().replicate(Index(3)).matrix(), horizontal); + VERIFY_IS_EQUAL(m.rowwise().replicate(fix<1>).matrix(), m.matrix()); + VERIFY_IS_EQUAL(m.rowwise().replicate(fix<0>).cols(), 0); + + STATIC_CHECK((std::is_same<decltype(m.colwise().replicate(fix<3>)), const Replicate<ExpressionType, 3, 1>>::value)); + STATIC_CHECK( + (std::is_same<decltype(m.colwise().replicate(fix<3>(3))), const Replicate<ExpressionType, 3, 1>>::value)); + STATIC_CHECK((std::is_same<decltype(m.colwise().replicate(fix<Dynamic>(2))), + const Replicate<ExpressionType, Dynamic, 1>>::value)); + STATIC_CHECK((std::is_same<decltype(m.colwise().replicate(3)), const Replicate<ExpressionType, Dynamic, 1>>::value)); + STATIC_CHECK( + (std::is_same<decltype(m.colwise().replicate(Index(3))), const Replicate<ExpressionType, Dynamic, 1>>::value)); + STATIC_CHECK((std::is_same<decltype(m.rowwise().replicate(fix<3>)), const Replicate<ExpressionType, 1, 3>>::value)); + STATIC_CHECK( + (std::is_same<decltype(m.rowwise().replicate(fix<3>(3))), const Replicate<ExpressionType, 1, 3>>::value)); + STATIC_CHECK((std::is_same<decltype(m.rowwise().replicate(fix<Dynamic>(2))), + const Replicate<ExpressionType, 1, Dynamic>>::value)); + STATIC_CHECK((std::is_same<decltype(m.rowwise().replicate(3)), const Replicate<ExpressionType, 1, Dynamic>>::value)); + STATIC_CHECK( + (std::is_same<decltype(m.rowwise().replicate(Index(3))), const Replicate<ExpressionType, 1, Dynamic>>::value)); +} + +void vectorwise_replicate_flexible_indexers() { + Matrix<int, 2, 3, ColMajor> m; + m << 1, 2, 3, 4, 5, 6; + vectorwise_replicate_indexers(m); + const auto& const_m = m; + vectorwise_replicate_indexers(const_m); + Matrix<double, Dynamic, Dynamic, RowMajor> d = m.cast<double>(); + vectorwise_replicate_indexers(d); + Array<float, 2, 3, RowMajor> a = m.cast<float>().array(); + vectorwise_replicate_indexers(a); + auto block = m.block<2, 2>(0, 1); + vectorwise_replicate_indexers(block); + const auto sum = m + m; + vectorwise_replicate_indexers(sum); +} + +void replicate_flexible_indexers() { + using Mat = Matrix<int, 2, 2>; + Mat m; + m << 1, 2, 3, 4; + const Matrix<int, 4, 6> ref = m.replicate<2, 3>(); + + VERIFY_IS_EQUAL(m.replicate(fix<2>, fix<3>), ref); + VERIFY_IS_EQUAL(m.replicate(fix<2>, 3), ref); + VERIFY_IS_EQUAL(m.replicate(fix<Dynamic>(2), fix<3>), ref); + VERIFY_IS_EQUAL(m.replicate(fix<2>(2), fix<3>(3)), ref); + VERIFY_IS_EQUAL(m.replicate(2, 3), ref); + VERIFY_IS_EQUAL(m.replicate(Index(2), Index(3)), ref); + STATIC_CHECK((std::is_same<std::decay_t<decltype(m.replicate(fix<2>, 3))>, Replicate<Mat, 2, Dynamic>>::value)); + STATIC_CHECK( + (std::is_same<std::decay_t<decltype(m.replicate(fix<Dynamic>(2), fix<3>))>, Replicate<Mat, Dynamic, 3>>::value)); + STATIC_CHECK((std::is_same<std::decay_t<decltype(m.replicate(2, 3))>, Replicate<Mat, Dynamic, Dynamic>>::value)); + STATIC_CHECK( + (std::is_same<std::decay_t<decltype(m.replicate(Index(2), Index(3)))>, Replicate<Mat, Dynamic, Dynamic>>::value)); +} + // A Replicate packet cannot cross a replication boundary, so the evaluator may serve packets // exactly when the inner (storage-order) direction is not replicated; without the flag, colwise // and rowwise broadcast operations silently fall back to scalar traversal. @@ -126,6 +205,8 @@ } EIGEN_DECLARE_TEST(array_replicate) { + CALL_SUBTEST_1(replicate_flexible_indexers()); + CALL_SUBTEST_1(vectorwise_replicate_flexible_indexers()); for (int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(replicate(Matrix<float, 1, 1>())); CALL_SUBTEST_2(replicate(Vector2f()));
diff --git a/test/basicstuff.cpp b/test/basicstuff.cpp index aab0f8e..80bd7d8 100644 --- a/test/basicstuff.cpp +++ b/test/basicstuff.cpp
@@ -276,6 +276,7 @@ } }; +template <typename = void> void casting_all() { casting_test_runner<bool>::run(); casting_test_runner<int8_t>::run(); @@ -366,7 +367,7 @@ CALL_SUBTEST_6(basicStuff(Matrix<float, 100, 100>())); CALL_SUBTEST_7(basicStuff(Matrix<long double, Dynamic, Dynamic>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE), internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); - CALL_SUBTEST_8(casting_all()); + CALL_SUBTEST_8(casting_all<>()); CALL_SUBTEST_3(basicStuffComplex( MatrixXcf(internal::random<int>(1, EIGEN_TEST_MAX_SIZE), internal::random<int>(1, EIGEN_TEST_MAX_SIZE))));
diff --git a/test/bdcsvd.cpp b/test/bdcsvd.cpp index 350e882..f152727 100644 --- a/test/bdcsvd.cpp +++ b/test/bdcsvd.cpp
@@ -53,6 +53,7 @@ #if defined(EIGEN_TEST_PART_46) || defined(EIGEN_TEST_PART_47) || defined(EIGEN_TEST_PART_48) || \ defined(EIGEN_TEST_PART_49) || defined(EIGEN_TEST_PART_ALL) // Verifies total deflation is **not** triggered. +template <typename = void> void compare_bdc_jacobi_instance(bool structure_as_m, int algoswap = 16) { MatrixXd m(4, 3); if (structure_as_m) { @@ -71,6 +72,7 @@ template <typename MatrixType> void bdcsvd_thin_full_options(const MatrixType& input = MatrixType()) { svd_thin_full_option_checks<MatrixType, 0>(input); + svd_thin_full_option_checks<MatrixType, PreconditionSquareMatrix>(input); } template <typename MatrixType> @@ -90,14 +92,14 @@ } // Verify SVD of bidiagonal matrix given as diagonal + superdiagonal vectors. -template <typename RealScalar> +template <typename RealScalar, int Options = ComputeFullU | ComputeFullV> void verify_bidiagonal_svd(const Matrix<RealScalar, Dynamic, 1>& diag, const Matrix<RealScalar, Dynamic, 1>& superdiag) { typedef Matrix<RealScalar, Dynamic, Dynamic> MatrixXr; typedef Matrix<RealScalar, Dynamic, 1> VectorXr; const Index n = diag.size(); - BDCSVD<MatrixXr, ComputeFullU | ComputeFullV> bdcsvd(diag, superdiag); + BDCSVD<MatrixXr, Options> bdcsvd(diag, superdiag); VERIFY(bdcsvd.info() == Success); const VectorXr& sv = bdcsvd.singularValues(); @@ -129,7 +131,7 @@ } // Verify that bidiagonal API and matrix API produce matching singular values. -template <typename RealScalar> +template <typename RealScalar, int Options = 0> void verify_bidiagonal_vs_matrix_svd(const Matrix<RealScalar, Dynamic, 1>& diag, const Matrix<RealScalar, Dynamic, 1>& superdiag) { typedef Matrix<RealScalar, Dynamic, Dynamic> MatrixXr; @@ -140,8 +142,8 @@ B.diagonal() = diag; if (n > 1) B.diagonal(1) = superdiag; - BDCSVD<MatrixXr> bidiag_svd(diag, superdiag); - BDCSVD<MatrixXr> matrix_svd(B); + BDCSVD<MatrixXr, Options> bidiag_svd(diag, superdiag); + BDCSVD<MatrixXr, Options> matrix_svd(B); VERIFY(bidiag_svd.info() == Success); VERIFY(matrix_svd.info() == Success); @@ -155,8 +157,10 @@ // Use the shared tridiagonal test matrix generators. // Each generator fills (diag, offdiag) which we treat as (diagonal, superdiagonal) // of a bidiagonal matrix. - test::for_all_tridiag_test_matrices<RealScalar>( - [](const auto& diag, const auto& offdiag) { verify_bidiagonal_svd<RealScalar>(diag, offdiag); }); + test::for_all_tridiag_test_matrices<RealScalar>([](const auto& diag, const auto& offdiag) { + verify_bidiagonal_svd<RealScalar>(diag, offdiag); + verify_bidiagonal_svd<RealScalar, PreconditionSquareMatrix | ComputeFullU | ComputeFullV>(diag, offdiag); + }); // Additional SVD-specific test: identity with cross-validation against full matrix SVD. test::for_tridiag_sizes<RealScalar>([](auto& diag, auto& offdiag) { @@ -174,6 +178,7 @@ } #if defined(EIGEN_TEST_PART_6) || defined(EIGEN_TEST_PART_ALL) +template <typename = void> void bdcsvd_mixed_option_enum_regression() { using NoQrFullSVD = BDCSVD<MatrixXd, NoQRPreconditioner | ComputeFullU | ComputeFullV>; using ReversedMixedSVD = BDCSVD<MatrixXd, ComputeThinU | DisableQRDecomposition | ComputeFullV>; @@ -183,10 +188,58 @@ STATIC_CHECK((int(ReversedMixedSVD::QRDecomposition) == int(DisableQRDecomposition))); STATIC_CHECK((ReversedMixedSVD::ComputationOptions == (ComputeThinU | ComputeFullV))); + + // Runtime unitary options must reach the JacobiSVD used below the switch size when Options holds only QR bits. + const MatrixXd m = MatrixXd::Random(5, 5); + EIGEN_DIAGNOSTICS(push) + EIGEN_DISABLE_DEPRECATED_WARNING + const BDCSVD<MatrixXd, DisableQRDecomposition> svd(m, ComputeFullU | ComputeFullV); + EIGEN_DIAGNOSTICS(pop) + svd_check_full(m, svd); +} +#endif + +#if defined(EIGEN_TEST_PART_65) || defined(EIGEN_TEST_PART_ALL) +template <typename = void> +void bdcsvd_switch_size() { + const MatrixXd m = MatrixXd::Random(20, 20); + BDCSVD<MatrixXd, ComputeFullU | ComputeFullV> svd(m.rows(), m.cols()); + svd.setSwitchSize(32); + svd.compute(m); + svd_check_full(m, svd); + svd.setSwitchSize(4); + svd.compute(m); + svd_check_full(m, svd); + svd.setSwitchSize(32); + svd.compute(m); + svd_check_full(m, svd); + + // The bidiagonal entry point allocates through the same path. + const VectorXd diag = VectorXd::Random(20); + const VectorXd superdiag = VectorXd::Random(19); + MatrixXd bidiag = MatrixXd::Zero(20, 20); + bidiag.diagonal() = diag; + bidiag.diagonal(1) = superdiag; + BDCSVD<MatrixXd, ComputeFullU | ComputeFullV> bidiag_svd(bidiag.rows(), bidiag.cols()); + bidiag_svd.setSwitchSize(32); + bidiag_svd.compute(diag, superdiag); + svd_check_full(bidiag, bidiag_svd); +} +#endif + +#if defined(EIGEN_TEST_PART_62) || defined(EIGEN_TEST_PART_ALL) +// Below the switch size, the bidiagonal API forwards PreconditionSquareMatrix to BDCSVD's JacobiSVD. +template <typename RealScalar> +void bdcsvd_precondition_square_matrix_bidiagonal(Index n) { + using VectorXr = Matrix<RealScalar, Dynamic, 1>; + const VectorXr diagonal = VectorXr::Random(n); + const VectorXr superdiagonal = VectorXr::Random(n - 1); + verify_bidiagonal_vs_matrix_svd<RealScalar, PreconditionSquareMatrix>(diagonal, superdiagonal); } #endif #if defined(EIGEN_TEST_PART_53) || defined(EIGEN_TEST_PART_ALL) +template <typename = void> void bdcsvd_extreme_scale_regressions() { typedef Matrix<double, 6, 6> Matrix6d; const double kTolerance = 16 * Matrix6d::RowsAtCompileTime * NumTraits<double>::epsilon(); @@ -239,6 +292,7 @@ #endif #if defined(EIGEN_TEST_PART_54) || defined(EIGEN_TEST_PART_ALL) +template <typename = void> void bdcsvd_fast_math_regression_1588() { const Index n = 500; MatrixXd matrix = MatrixXd::Zero(n, n); @@ -261,6 +315,51 @@ } #endif +// An all-subnormal matrix, which a SIMD unit that flushes subnormal inputs (ARMv7 NEON, Arm FZ, DAZ) reads as zero in +// the maxCoeff() selecting the scale: recovering the maximum from the representation and scaling through integer +// significands keeps the decomposition, and the singular values come back through the subnormal range exactly. The +// switch size below the dimensions keeps the divide-and-conquer path rather than the JacobiSVD fallback. +template <typename MatrixType> +void bdcsvd_flushed_subnormal_matrix(Index rows, Index cols) { + using RealScalar = typename MatrixType::RealScalar; + const MatrixType m = svd_subnormal_fixture<MatrixType>(rows, cols); + const int k = 60; + const MatrixType ms = m.unaryExpr(internal::scale_by_exponent_op<RealScalar>(k)); + BDCSVD<MatrixType, ComputeFullU | ComputeFullV> scaled; + scaled.setSwitchSize(8); + scaled.compute(ms); + forEachFlushToZeroMode([&](FlushToZeroMode) { + BDCSVD<MatrixType, ComputeFullU | ComputeFullV> svd; + svd.setSwitchSize(8); + svd.compute(m); + svd_check_flushed_subnormal(svd, scaled, m, k); + }); +} + +// The bidiagonal entry point, on both sides of the switch size: divide and conquer above it, JacobiSVD below. +template <typename RealScalar> +void bdcsvd_flushed_subnormal_bidiagonal(Index n, int switchSize) { + using Vec = Matrix<RealScalar, Dynamic, 1>; + using Mat = Matrix<RealScalar, Dynamic, Dynamic>; + const Vec diagonal = svd_subnormal_fixture<Vec>(n, 1), superdiagonal = svd_subnormal_fixture<Vec>(n - 1, 1); + const int k = 60; + const Vec ds = diagonal.unaryExpr(internal::scale_by_exponent_op<RealScalar>(k)); + const Vec ss = superdiagonal.unaryExpr(internal::scale_by_exponent_op<RealScalar>(k)); + Mat B = Mat::Zero(n, n); + B.diagonal() = diagonal; + B.diagonal(1) = superdiagonal; + BDCSVD<Mat, ComputeFullU | ComputeFullV> scaled; + scaled.setSwitchSize(switchSize); + scaled.compute(ds, ss); + forEachFlushToZeroMode([&](FlushToZeroMode) { + BDCSVD<Mat, ComputeFullU | ComputeFullV> svd; + svd.setSwitchSize(switchSize); + svd.compute(diagonal, superdiagonal); + svd_check_flushed_subnormal(svd, scaled, B, k); + }); +} + +template <typename = void> void bdcsvd_power_of_two_scaling() { // Reciprocal scaling rounds the smaller singular value down by one ULP in both entry paths. const Index size = 20; @@ -384,7 +483,7 @@ CALL_SUBTEST_3((bdcsvd_asserts<Matrix<float, 10, 7>>())); CALL_SUBTEST_4((bdcsvd_asserts<Matrix<float, 7, 10>>())); CALL_SUBTEST_5((bdcsvd_asserts<Matrix<std::complex<double>, 6, 9>>())); - CALL_SUBTEST_6((bdcsvd_mixed_option_enum_regression())); + CALL_SUBTEST_6((bdcsvd_mixed_option_enum_regression<>())); CALL_SUBTEST_7((bdcsvd_thin_full_options<Matrix2cd>())); CALL_SUBTEST_9((bdcsvd_thin_full_options<Matrix2d>())); @@ -428,6 +527,30 @@ CALL_SUBTEST_41( (svd_check_max_size_matrix<Matrix<float, Dynamic, Dynamic, RowMajor, 35, 20>, HouseholderQRPreconditioner>(r, c))); + + const Index smallSize = internal::random<Index>(2, 15); + TEST_SET_BUT_UNUSED_VARIABLE(smallSize); + CALL_SUBTEST_62((bdcsvd_precondition_square_matrix_bidiagonal<double>(smallSize))); + CALL_SUBTEST_62((bdcsvd_precondition_square_matrix_bidiagonal<float>(smallSize))); + CALL_SUBTEST_62((svd_precondition_square_matrix_accuracy<Matrix2f>(2))); + CALL_SUBTEST_62((svd_precondition_square_matrix_accuracy<Matrix4d>(4))); + CALL_SUBTEST_62((svd_precondition_square_matrix_accuracy<MatrixXf>(12))); + CALL_SUBTEST_62((svd_precondition_square_matrix_accuracy<MatrixXd>(12))); + CALL_SUBTEST_62((svd_precondition_square_matrix_accuracy<Matrix<double, Dynamic, Dynamic, RowMajor>>(12))); + CALL_SUBTEST_63((svd_precondition_square_matrix_accuracy<Matrix2cd>(2))); + CALL_SUBTEST_63((svd_precondition_square_matrix_accuracy<Matrix4cf>(4))); + CALL_SUBTEST_63((svd_precondition_square_matrix_accuracy<MatrixXcf>(12))); + CALL_SUBTEST_63((svd_precondition_square_matrix_accuracy<MatrixXcd>(12))); + CALL_SUBTEST_64((svd_precondition_square_matrix_runtime_options<MatrixXf>(12))); + CALL_SUBTEST_64((svd_precondition_square_matrix_runtime_options<MatrixXcd>(internal::random<Index>(1, 30)))); + CALL_SUBTEST_64((svd_precondition_square_matrix_runtime_options<Matrix2f>(2))); + CALL_SUBTEST_64((bdcsvd_thin_full_options<Matrix<double, Dynamic, Dynamic, RowMajor>>( + Matrix<double, Dynamic, Dynamic, RowMajor>(8, 8)))); + // smallSvd keeps its QR preconditioner, so small non-square inputs work with DisableQRDecomposition too. + CALL_SUBTEST_64( + (svd_thin_full_option_checks<MatrixXd, DisableQRDecomposition | PreconditionSquareMatrix>(MatrixXd(10, 5)))); + CALL_SUBTEST_64( + (svd_thin_full_option_checks<MatrixXd, DisableQRDecomposition | PreconditionSquareMatrix>(MatrixXd(5, 10)))); } // test matrixbase method @@ -439,15 +562,18 @@ // CALL_SUBTEST_9( svd_preallocate<void>() ); CALL_SUBTEST_45(svd_underoverflow<void>()); - CALL_SUBTEST_45(bdcsvd_power_of_two_scaling()); + CALL_SUBTEST_45(bdcsvd_power_of_two_scaling<>()); + CALL_SUBTEST_45((svd_zero_matrix_solvers<Matrix3d>(3, 3))); + CALL_SUBTEST_45((svd_zero_matrix_solvers<Matrix<float, 2, 5>>(2, 5))); + CALL_SUBTEST_45((svd_zero_matrix_solvers<MatrixXcf>(5, 3))); // Without total deflation issues. - CALL_SUBTEST_46((compare_bdc_jacobi_instance(true))); - CALL_SUBTEST_47((compare_bdc_jacobi_instance(false))); + CALL_SUBTEST_46((compare_bdc_jacobi_instance<>(true))); + CALL_SUBTEST_47((compare_bdc_jacobi_instance<>(false))); // With total deflation issues before, when it shouldn't be triggered. - CALL_SUBTEST_48((compare_bdc_jacobi_instance(true, 3))); - CALL_SUBTEST_49((compare_bdc_jacobi_instance(false, 3))); + CALL_SUBTEST_48((compare_bdc_jacobi_instance<>(true, 3))); + CALL_SUBTEST_49((compare_bdc_jacobi_instance<>(false, 3))); // Convergence for large constant matrix (https://gitlab.com/libeigen/eigen/-/issues/2491) CALL_SUBTEST_50(bdcsvd_check_convergence<MatrixXf>(MatrixXf::Constant(500, 500, 1))); @@ -455,11 +581,18 @@ // Bidiagonal SVD hard test cases CALL_SUBTEST_51((bdcsvd_bidiagonal_hard_cases<float>())); CALL_SUBTEST_52((bdcsvd_bidiagonal_hard_cases<double>())); - CALL_SUBTEST_53((bdcsvd_extreme_scale_regressions())); - CALL_SUBTEST_54((bdcsvd_fast_math_regression_1588())); + CALL_SUBTEST_53((bdcsvd_extreme_scale_regressions<>())); + CALL_SUBTEST_54((bdcsvd_fast_math_regression_1588<>())); CALL_SUBTEST_55((bdcsvd_secular_extrapolation<float>())); CALL_SUBTEST_56((bdcsvd_secular_extrapolation<double>())); CALL_SUBTEST_57((bdcsvd_qr_crossover<float, ColMajor>())); CALL_SUBTEST_58((bdcsvd_qr_crossover<double, RowMajor>())); CALL_SUBTEST_59((bdcsvd_qr_crossover<std::complex<double>, ColMajor>())); + CALL_SUBTEST_60((bdcsvd_flushed_subnormal_matrix<MatrixXf>(12, 10))); + CALL_SUBTEST_60((bdcsvd_flushed_subnormal_matrix<MatrixXd>(10, 12))); + CALL_SUBTEST_60((bdcsvd_flushed_subnormal_matrix<MatrixXcd>(10, 10))); + CALL_SUBTEST_60((bdcsvd_flushed_subnormal_bidiagonal<float>(12, 8))); + CALL_SUBTEST_60((bdcsvd_flushed_subnormal_bidiagonal<double>(16, 8))); + CALL_SUBTEST_60((bdcsvd_flushed_subnormal_bidiagonal<double>(6, 8))); + CALL_SUBTEST_65((bdcsvd_switch_size<>())); }
diff --git a/test/bicgstab.cpp b/test/bicgstab.cpp index 2b3e5de..73f49ad 100644 --- a/test/bicgstab.cpp +++ b/test/bicgstab.cpp
@@ -83,10 +83,28 @@ VERIFY(residual.isZero()); } +// solveWithGuessInPlace() lives on IterativeSolverBase; check it on a second solver. +void test_solve_with_guess_in_place() { + const Index n = 40; + MatrixXd M = MatrixXd::Random(n, n); + SparseMatrix<double> A = (M + double(n) * MatrixXd::Identity(n, n)).sparseView(); + VectorXd b = VectorXd::Random(n), x0 = VectorXd::Random(n); + BiCGSTAB<SparseMatrix<double> > solver(A); + solver.setTolerance(1e-10); + VectorXd x1 = solver.solveWithGuess(b, x0); + VERIFY_IS_EQUAL(solver.info(), Success); + VectorXd x2 = x0; + solver.solveWithGuessInPlace(b, x2); + VERIFY_IS_EQUAL(solver.info(), Success); + VERIFY_IS_APPROX(x1, x2); + VERIFY((A * x2 - b).norm() <= 10 * solver.tolerance() * b.norm()); +} + EIGEN_DECLARE_TEST(bicgstab) { CALL_SUBTEST_1((test_bicgstab_T<double, int>())); CALL_SUBTEST_2((test_bicgstab_T<std::complex<double>, int>())); CALL_SUBTEST_3((test_bicgstab_T<double, long int>())); CALL_SUBTEST_4(test_2856()); CALL_SUBTEST_5(test_2899()); + CALL_SUBTEST_6(test_solve_with_guess_in_place()); }
diff --git a/test/bicgstabl.cpp b/test/bicgstabl.cpp index 256ce93..57ab63d 100644 --- a/test/bicgstabl.cpp +++ b/test/bicgstabl.cpp
@@ -25,6 +25,7 @@ CALL_SUBTEST(check_sparse_square_solving(bicgstabl_colmajor_ilut)); } +template <typename = void> void test_bicgstabl_solve_with_guess_restart() { // Regression for a solveWithGuess case where the recursive residual reached // the tolerance before the true residual did. @@ -35,5 +36,5 @@ EIGEN_DECLARE_TEST(bicgstabl) { CALL_SUBTEST_1(test_bicgstabl_T<double>()); CALL_SUBTEST_2(test_bicgstabl_T<std::complex<double> >()); - CALL_SUBTEST_3(test_bicgstabl_solve_with_guess_restart()); + CALL_SUBTEST_3(test_bicgstabl_solve_with_guess_restart<>()); }
diff --git a/test/block_coeffwise.cpp b/test/block_coeffwise.cpp new file mode 100644 index 0000000..59bf5c5 --- /dev/null +++ b/test/block_coeffwise.cpp
@@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include "main.h" + +template <typename Dst, typename Src, typename Functor> +void check_block_coeffwise_dispatch() { +#if EIGEN_UNALIGNED_VECTORIZE + using Scalar = typename Dst::Scalar; + using Kernel = internal::generic_dense_assignment_kernel<internal::evaluator<Dst>, internal::evaluator<Src>, Functor>; + using Traits = typename Kernel::AssignmentTraits; + constexpr bool packetSupported = internal::packet_traits<Scalar>::Vectorizable && + internal::packet_traits<Scalar>::HasMul && + internal::functor_traits<Functor>::PacketAccess; + STATIC_CHECK(!packetSupported || Traits::Traversal == SliceVectorizedTraversal); + STATIC_CHECK(!packetSupported || Traits::Unrolling == NoUnrolling); + using Loop = internal::dense_assignment_loop_impl<Kernel, SliceVectorizedTraversal, NoUnrolling>; + STATIC_CHECK(!packetSupported || EIGEN_MAX_ALIGN_BYTES < Loop::RequestedAlignment || Loop::DstIsAligned); +#endif +} + +template <typename Scalar, int Order> +void block_coeffwise() { + using Mat = Matrix<Scalar, Dynamic, Dynamic, Order>; + using Product = internal::remove_all_t< + decltype(std::declval<const Mat&>().block(0, 0, 0, 0).cwiseProduct(std::declval<const Mat&>()))>; + check_block_coeffwise_dispatch<Mat, Product, internal::assign_op<Scalar, Scalar>>(); + check_block_coeffwise_dispatch<Mat, Product, internal::add_assign_op<Scalar, Scalar>>(); + check_block_coeffwise_dispatch<Mat, Product, internal::sub_assign_op<Scalar, Scalar>>(); + constexpr Index packetSize = internal::packet_traits<Scalar>::size; + for (Index inner = 0; inner <= (std::max)(Index(33), 2 * packetSize + 1); ++inner) { + for (Index offset : {0, 1, 3}) { + const Index rows = Order == ColMajor ? inner : 5; + const Index cols = Order == ColMajor ? 5 : inner; + const Mat src = Mat::Random(rows + 7, cols + 7); + const Mat rhs = Mat::Random(rows, cols); + Mat dst = Mat::Constant(rows + 7, cols + 7, Scalar(3)); + Mat expected = dst; + auto block = dst.block(offset, offset, rows, cols); + block.array() = src.block(offset, offset, rows, cols).array() * rhs.array(); + for (Index j = 0; j < cols; ++j) + for (Index i = 0; i < rows; ++i) expected(i + offset, j + offset) = src(i + offset, j + offset) * rhs(i, j); + VERIFY_IS_APPROX(dst, expected); + block.array() += src.block(offset, offset, rows, cols).array() * rhs.array(); + expected.block(offset, offset, rows, cols) *= Scalar(2); + VERIFY_IS_APPROX(dst, expected); + Mat packed = src.block(offset, offset, rows, cols).cwiseProduct(rhs); + VERIFY_IS_APPROX(packed, (expected.block(offset, offset, rows, cols) / Scalar(2))); + const auto product = src.block(offset, offset, rows, cols).cwiseProduct(rhs); + const Mat initial = Mat::Random(rows, cols); + Mat compound = initial; + Mat reference(rows, cols); + compound += product; + for (Index j = 0; j < cols; ++j) + for (Index i = 0; i < rows; ++i) reference(i, j) = initial(i, j) + src(i + offset, j + offset) * rhs(i, j); + VERIFY_IS_APPROX(compound, reference); + compound = initial; + compound -= product; + for (Index j = 0; j < cols; ++j) + for (Index i = 0; i < rows; ++i) reference(i, j) = initial(i, j) - src(i + offset, j + offset) * rhs(i, j); + VERIFY_IS_APPROX(compound, reference); + } + } +} + +EIGEN_DECLARE_TEST(block_coeffwise) { + CALL_SUBTEST((block_coeffwise<float, ColMajor>())); + CALL_SUBTEST((block_coeffwise<float, RowMajor>())); + CALL_SUBTEST((block_coeffwise<double, ColMajor>())); + CALL_SUBTEST((block_coeffwise<double, RowMajor>())); + CALL_SUBTEST((block_coeffwise<std::complex<float>, ColMajor>())); + CALL_SUBTEST((block_coeffwise<std::complex<float>, RowMajor>())); +}
diff --git a/test/buildsystem/CMakeLists.txt b/test/buildsystem/CMakeLists.txt index 589f317..f2197f5 100644 --- a/test/buildsystem/CMakeLists.txt +++ b/test/buildsystem/CMakeLists.txt
@@ -33,6 +33,7 @@ install_off_all_options exclude_from_all suitesparse_reconfigure + test_part_groups ) if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21)
diff --git a/test/buildsystem/consumers/gpu_compile_options/CMakeLists.txt b/test/buildsystem/consumers/gpu_compile_options/CMakeLists.txt index 6d96880..339db03 100644 --- a/test/buildsystem/consumers/gpu_compile_options/CMakeLists.txt +++ b/test/buildsystem/consumers/gpu_compile_options/CMakeLists.txt
@@ -22,8 +22,8 @@ set(CMAKE_CUDA_COMPILER_ID "${compiler}") set(EIGEN_ADD_TEST_FILENAME_EXTENSION "${extension}") set(MSVC "${msvc}") - set(EIGEN_TEST_CUSTOM_CXX_FLAGS "/d2ReducedOptimizeHugeFunctions;/DEIGEN_STRONG_INLINE=inline;/Os;-DKEEP=1") - ei_add_test_internal("${CMAKE_CURRENT_BINARY_DIR}/dummy" "${name}" "/W4 /DLOCAL_DEF=2") + set(EIGEN_TEST_CUSTOM_CXX_FLAGS "${custom_flags}") + ei_add_test_internal("${CMAKE_CURRENT_BINARY_DIR}/dummy" "${name}" "${test_flags}") get_target_property(options "${name}" COMPILE_OPTIONS) get_target_property(definitions "${name}" COMPILE_DEFINITIONS) if(NOT options STREQUAL expected_options) @@ -34,14 +34,25 @@ endif() endfunction() -set(msvc_options "/bigobj;/W4;/DLOCAL_DEF=2;/d2ReducedOptimizeHugeFunctions;/DEIGEN_STRONG_INLINE=inline;/Os;-DKEEP=1") -set(nvcc_options "-Xcompiler=/bigobj;-Xcompiler=/W4;-Xcompiler=/d2ReducedOptimizeHugeFunctions;-Xcompiler=/Os;-DKEEP=1") +set(custom_flags "/d2ReducedOptimizeHugeFunctions;/DEIGEN_STRONG_INLINE=inline;/Os;-DKEEP=1") +set(test_flags "/W4 /DLOCAL_DEF=2") +set(msvc_options "-bigobj;/W4;/DLOCAL_DEF=2;/d2ReducedOptimizeHugeFunctions;/DEIGEN_STRONG_INLINE=inline;/Os;-DKEEP=1") +set(nvcc_options "-Xcompiler=-bigobj;-Xcompiler=-W4;-Xcompiler=-d2ReducedOptimizeHugeFunctions;-Xcompiler=-Os") set(definitions "EIGEN_TEST_MAX_SIZE=320") check_options(nvcc cuda-language NVIDIA cu TRUE "${nvcc_options}" - "${definitions};LOCAL_DEF=2;EIGEN_STRONG_INLINE=inline") + "${definitions};LOCAL_DEF=2;EIGEN_STRONG_INLINE=inline;KEEP=1") check_options(host cuda-language NVIDIA cpp TRUE "${msvc_options}" "${definitions}") check_options(clang cuda-language Clang cu TRUE "${msvc_options}" "${definitions}") check_options(cuda_as_cxx cuda-as-cxx NVIDIA cu TRUE "${msvc_options}" "${definitions}") check_options(non_msvc cuda-language NVIDIA cu FALSE "/W4;/DLOCAL_DEF=2;/d2ReducedOptimizeHugeFunctions;/DEIGEN_STRONG_INLINE=inline;/Os;-DKEEP=1" "${definitions}") + +# The spelling CI passes, which sccache can cache: nvcc still gets every -option as a host flag. +set(custom_flags "-d2ReducedOptimizeHugeFunctions;-DEIGEN_STRONG_INLINE=inline;-Os") +set(test_flags "-W4 -DLOCAL_DEF=2") +check_options(nvcc_dash cuda-language NVIDIA cu TRUE "${nvcc_options}" + "${definitions};LOCAL_DEF=2;EIGEN_STRONG_INLINE=inline") +check_options(host_dash cuda-language NVIDIA cpp TRUE + "-bigobj;-W4;-DLOCAL_DEF=2;-d2ReducedOptimizeHugeFunctions;-DEIGEN_STRONG_INLINE=inline;-Os" + "${definitions}")
diff --git a/test/buildsystem/consumers/test_part_groups/CMakeLists.txt b/test/buildsystem/consumers/test_part_groups/CMakeLists.txt new file mode 100644 index 0000000..5c0657b --- /dev/null +++ b/test/buildsystem/consumers/test_part_groups/CMakeLists.txt
@@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: The Eigen Authors +# SPDX-License-Identifier: MPL-2.0 + +# Registers a five-part test through ei_add_test with parts 1-3 grouped and +# checks the resulting targets. PART_GROUPS_CASE selects a variant that must +# fail to configure instead: "smoke" names a grouped part in the smoke-test +# list, "explicit" groups a source with an EIGEN_TEST_PART_<N> marker. + +cmake_minimum_required(VERSION 3.18) +project(TestPartGroups CXX) +include("${EIGEN_SOURCE_DIR}/cmake/EigenTesting.cmake") + +add_library(Eigen3::Eigen INTERFACE IMPORTED) +add_custom_target(buildtests) +add_custom_target(buildsmoketests) +set(EIGEN_TEST_MAX_SIZE 320) +set(EIGEN_SPLIT_LARGE_TESTS ON) +set(ei_smoke_test_list grouped_5) +set(ei_test_part_groups_grouped "1-3") + +set(body "CALL_SUBTEST_1 CALL_SUBTEST_2 CALL_SUBTEST_3 CALL_SUBTEST_4 CALL_SUBTEST_5\n") +if(PART_GROUPS_CASE STREQUAL "smoke") + list(APPEND ei_smoke_test_list grouped_2) +elseif(PART_GROUPS_CASE STREQUAL "explicit") + string(APPEND body "#ifdef EIGEN_TEST_PART_2\n#endif\n") +endif() + +# ei_add_test reads <testname>.cpp relative to the current source directory, so +# the test and its registration live in a generated subdirectory. +set(gen "${CMAKE_CURRENT_BINARY_DIR}/gen") +file(WRITE "${gen}/grouped.cpp" "${body}int main() { return 0; }\n") +file(WRITE "${gen}/CMakeLists.txt" "ei_add_test(grouped)\n") +add_subdirectory("${gen}" gen) + +function(expect_part_defines target expected) + if(NOT TARGET ${target}) + message(FATAL_ERROR "missing target ${target}") + endif() + get_target_property(definitions ${target} COMPILE_DEFINITIONS) + list(FILTER definitions INCLUDE REGEX "^EIGEN_TEST_PART_") + if(NOT definitions STREQUAL expected) + message(FATAL_ERROR "${target}: expected [${expected}], got [${definitions}]") + endif() +endfunction() + +expect_part_defines(grouped_1 "EIGEN_TEST_PART_1=1;EIGEN_TEST_PART_2=1;EIGEN_TEST_PART_3=1") +expect_part_defines(grouped_4 "EIGEN_TEST_PART_4=1") +expect_part_defines(grouped_5 "EIGEN_TEST_PART_5=1") +foreach(absorbed grouped_2 grouped_3) + if(TARGET ${absorbed}) + message(FATAL_ERROR "${absorbed} should be compiled as part of grouped_1") + endif() +endforeach() +get_target_property(parent_deps grouped MANUALLY_ADDED_DEPENDENCIES) +if(NOT parent_deps STREQUAL "grouped_1;grouped_4;grouped_5") + message(FATAL_ERROR "grouped: expected dependencies [grouped_1;grouped_4;grouped_5], got [${parent_deps}]") +endif() +get_target_property(smoke_deps buildsmoketests MANUALLY_ADDED_DEPENDENCIES) +if(NOT smoke_deps STREQUAL "grouped_5") + message(FATAL_ERROR "buildsmoketests: expected [grouped_5], got [${smoke_deps}]") +endif()
diff --git a/test/buildsystem/scenarios/test_part_groups.cmake b/test/buildsystem/scenarios/test_part_groups.cmake new file mode 100644 index 0000000..6f9a740 --- /dev/null +++ b/test/buildsystem/scenarios/test_part_groups.cmake
@@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: The Eigen Authors +# SPDX-License-Identifier: MPL-2.0 + +# ei_add_test compiles the part ranges listed for a test as one executable, and +# refuses a grouping that would change what a part compiles or drop a smoke test. + +bs_configure("grouped parts" "${BS_CONSUMER_DIR}/test_part_groups" "${WORK_DIR}/grouped" + "-DEIGEN_SOURCE_DIR=${EIGEN_SOURCE_DIR}") + +bs_configure_expect_failure("grouped smoke part" "${BS_CONSUMER_DIR}/test_part_groups" + "${WORK_DIR}/smoke" output + "-DEIGEN_SOURCE_DIR=${EIGEN_SOURCE_DIR}" "-DPART_GROUPS_CASE=smoke") +if(NOT output MATCHES "grouped_2 is a smoke test and cannot be grouped") + bs_fail("configure failed for some reason other than the smoke-test check\n----\n${output}\n----") +endif() + +bs_configure_expect_failure("grouped explicit parts" "${BS_CONSUMER_DIR}/test_part_groups" + "${WORK_DIR}/explicit" output + "-DEIGEN_SOURCE_DIR=${EIGEN_SOURCE_DIR}" "-DPART_GROUPS_CASE=explicit") +if(NOT output MATCHES "parts with EIGEN_TEST_PART_<N> markers cannot be grouped") + bs_fail("configure failed for some reason other than the marker check\n----\n${output}\n----") +endif()
diff --git a/test/bunchkaufman.cpp b/test/bunchkaufman.cpp index f037131..d246b92 100644 --- a/test/bunchkaufman.cpp +++ b/test/bunchkaufman.cpp
@@ -317,6 +317,7 @@ // det(D) can be representable while the block determinants it is built from are not. The blocks below are // [[s/2, s], [s, s/2]] with det = -3s^2/4 at s = 2^600 and s = 2^-600, one overflowing and one // underflowing, whose product is exactly 9/16. Multiplying the blocks directly gives inf * 0 = NaN. +template <typename = void> void bunchkaufman_determinant_mixed_scale() { MatrixXd a = MatrixXd::Zero(4, 4); for (int b = 0; b < 2; ++b) { @@ -392,6 +393,7 @@ // row, which can dwarf |d21|. So d22/d21 can overflow, and the scaled determinant comes out +-inf, or 0*inf // = NaN where d11 is zero; +inf and NaN count the block as definite with the sign of its trace, which is // positive here. The criterion puts the true magnitude below 1, which is what rejects all three. +template <typename = void> void bunchkaufman_inertia_wide_2x2_block() { // The 2x2 block is (d11, d21, d22) = (a00, 1e-10, 1e299), determinant a00*1e299 - 1e-20 < 0 for every // a00 below, so it contributes one eigenvalue of each sign. The block's inverse has entries of order @@ -414,6 +416,7 @@ // The scaling above divides by numext::abs(d21), so it relies on that magnitude not underflowing: for // |3u + 4iu| Eigen's hypot scales by the larger component instead of summing squares, giving 5u exactly // where u^2 would be zero. Not template code, because building the off-diagonal needs a complex literal. +template <typename = void> void bunchkaufman_determinant_subnormal_block_complex() { typedef std::complex<double> Scalar; @@ -481,7 +484,8 @@ typedef typename NumTraits<Scalar>::Real RealScalar; typedef Matrix<Scalar, 2, 2> Mat2; typedef Matrix<Scalar, 2, 1> Vec2; - const RealScalar tol = numext::sqrt(test_precision<RealScalar>()); + // One reciprocal and a 2x2 product; include complex multiply/add rounding. + const RealScalar tol = RealScalar(8) * NumTraits<RealScalar>::epsilon(); for (RealScalar mag : {numext::pow(RealScalar(10), RealScalar(200)), numext::pow(RealScalar(10), RealScalar(-200))}) { const Scalar off = Scalar(mag); Mat2 A; @@ -492,11 +496,11 @@ VERIFY(!bk.isNegative()); // Use the max-abs (infinity) relative norm throughout: the Frobenius norm (.norm()) squares the // ~1e200 entries and would overflow/underflow even for a correct factorization. - VERIFY((A - bk.reconstructedMatrix()).cwiseAbs().maxCoeff() <= tol * A.cwiseAbs().maxCoeff()); + VERIFY(max_abs_coeff(A - bk.reconstructedMatrix()) <= tol * max_abs_coeff(A)); // A x = b with b = [1,1]; the product A*x stays O(1), so its residual is safe to measure. const Vec2 b(Scalar(1), Scalar(1)); const Vec2 x = bk.solve(b); - VERIFY((A * x - b).cwiseAbs().maxCoeff() <= tol); + VERIFY(max_abs_coeff(A * x - b) <= tol); } } @@ -737,12 +741,12 @@ // Subnormal 2x2 block: the determinant underflows, its log and sign do not. CALL_SUBTEST_5(bunchkaufman_determinant_subnormal_block<MatrixXd>()); CALL_SUBTEST_6(bunchkaufman_determinant_subnormal_block<MatrixXcd>()); - CALL_SUBTEST_6(bunchkaufman_determinant_subnormal_block_complex()); + CALL_SUBTEST_6(bunchkaufman_determinant_subnormal_block_complex<>()); CALL_SUBTEST_8(bunchkaufman_determinant_subnormal_block<MatrixXf>()); - CALL_SUBTEST_5(bunchkaufman_inertia_wide_2x2_block()); + CALL_SUBTEST_5(bunchkaufman_inertia_wide_2x2_block<>()); // Mixed-scale 2x2 blocks: the block determinants leave the representable range, their product does not. - CALL_SUBTEST_5(bunchkaufman_determinant_mixed_scale()); + CALL_SUBTEST_5(bunchkaufman_determinant_mixed_scale<>()); // Problem-size constructors. CALL_SUBTEST_8(BunchKaufman<MatrixXf>(10));
diff --git a/test/conjugate_gradient.cpp b/test/conjugate_gradient.cpp index dc0c782..3a2f45f 100644 --- a/test/conjugate_gradient.cpp +++ b/test/conjugate_gradient.cpp
@@ -36,6 +36,7 @@ LeastSquaresConjugateGradient<MatrixType> lscg; } +template <typename = void> void test_conjugate_gradient_extreme_rhs() { const Matrix2d mat = Matrix2d::Identity(); const Vector2d direction = (Vector2d() << 1, -1).finished(); @@ -57,10 +58,32 @@ } } +// solveWithGuessInPlace() must agree with solveWithGuess() and report through info(). +template <typename = void> +void test_solve_with_guess_in_place() { + const Index n = 60; + MatrixXd M = MatrixXd::Random(n, n); + SparseMatrix<double> A = (M * M.transpose() + double(n) * MatrixXd::Identity(n, n)).sparseView(); + VectorXd b = VectorXd::Random(n), x0 = VectorXd::Random(n); + ConjugateGradient<SparseMatrix<double>, Lower | Upper> cg(A); + cg.setTolerance(1e-12); + VectorXd x1 = cg.solveWithGuess(b, x0); + VERIFY_IS_EQUAL(cg.info(), Success); + VectorXd x2 = x0; + cg.solveWithGuessInPlace(b, x2); + VERIFY_IS_EQUAL(cg.info(), Success); + VERIFY(cg.iterations() > 0); + VERIFY_IS_APPROX(x1, x2); + // The recursive residual is below tolerance() ||b|| at exit; the true residual differs by + // the rounding of a few iterations, far below the factor 10. + VERIFY((A * x2 - b).norm() <= 10 * cg.tolerance() * b.norm()); +} + EIGEN_DECLARE_TEST(conjugate_gradient) { CALL_SUBTEST_1((test_conjugate_gradient_T<double, int>())); CALL_SUBTEST_2((test_conjugate_gradient_T<std::complex<double>, int>())); CALL_SUBTEST_3((test_conjugate_gradient_T<double, long int>())); CALL_SUBTEST_4(test_default_construct_fixed_size<Matrix3d>()); - CALL_SUBTEST_5(test_conjugate_gradient_extreme_rhs()); + CALL_SUBTEST_5(test_conjugate_gradient_extreme_rhs<>()); + CALL_SUBTEST_6(test_solve_with_guess_in_place<>()); }
diff --git a/test/dense_storage.cpp b/test/dense_storage.cpp index 898ed89..4c86d33 100644 --- a/test/dense_storage.cpp +++ b/test/dense_storage.cpp
@@ -16,6 +16,7 @@ #include "SafeScalar.h" #include <Eigen/Core> +#include <new> using DenseStorageD3x3 = Eigen::DenseStorage<double, 9, 3, 3, 0>; #if !defined(EIGEN_DENSE_STORAGE_CTOR_PLUGIN) @@ -251,12 +252,105 @@ VERIFY_IS_CWISE_EQUAL(m1, m0); } +// Building a fixed-maximum-size object from an expression must not value-initialize its storage: that zero-fills the +// whole inline buffer whatever the logical size (#3179). The object is constructed over a sentinel pattern and the +// bytes past the logical size are checked to still hold it. Aggregate member initialization is copy-initialization, +// which selects the implicit converting constructors rather than the explicit one-argument one. The sentinel is written +// and read through volatile so that the optimizer keeps both accesses. Only scalars with a trivial default constructor +// can be probed this way: std::complex, for instance, zero-initializes itself on default-initialization. +template <typename PlainType> +struct plain_holder { + PlainType m; +}; + +template <typename PlainType> +struct plain_storage { + alignas(PlainType) unsigned char bytes[sizeof(PlainType)]; + void fill() { + volatile unsigned char* v = bytes; + for (std::size_t i = 0; i < sizeof(bytes); ++i) v[i] = 0xAB; + } +}; + +template <typename PlainType> +void verify_unused_storage_untouched(const PlainType& m) { + const volatile unsigned char* p = reinterpret_cast<const unsigned char*>(m.data() + m.size()); + const volatile unsigned char* end = + reinterpret_cast<const unsigned char*>(m.data() + PlainType::MaxSizeAtCompileTime); + VERIFY(p < end); + for (; p != end; ++p) VERIFY_IS_EQUAL(int(*p), 0xAB); +} + +template <typename PlainType, typename Expr, typename Reference> +void check_unused_storage_untouched(const Expr& expr, const Reference& reference) { + plain_storage<PlainType> storage; + storage.fill(); + plain_holder<PlainType>* holder = new (storage.bytes) plain_holder<PlainType>{expr}; + VERIFY_IS_CWISE_EQUAL(holder->m, reference); + verify_unused_storage_untouched(holder->m); +} + +void fixed_max_storage_tests() { + using MatrixMax = Matrix<double, Dynamic, Dynamic, ColMajor, 8, 8>; + using RowMatrixMax = Matrix<float, Dynamic, Dynamic, RowMajor, 8, 8>; + using ArrayMax = Array<int, 4, Dynamic, ColMajor, 4, 8>; + using VectorMax = Matrix<int, Dynamic, 1, ColMajor, 16, 1>; + MatrixMax matrix = MatrixMax::Random(8, 8); + RowMatrixMax row_matrix = RowMatrixMax::Random(8, 8); + ArrayMax array = ArrayMax::Random(4, 8); + VectorMax vector = VectorMax::Random(16); + // conversion from a DenseBase expression + check_unused_storage_untouched<MatrixMax>(matrix.topLeftCorner(3, 5), matrix.topLeftCorner(3, 5)); + check_unused_storage_untouched<RowMatrixMax>(row_matrix.bottomRows(6) * 2.0f, row_matrix.bottomRows(6) * 2.0f); + check_unused_storage_untouched<ArrayMax>(array.leftCols(3) + 1, array.leftCols(3) + 1); + check_unused_storage_untouched<VectorMax>(vector.head(5), vector.head(5)); + // conversion from an EigenBase expression + check_unused_storage_untouched<MatrixMax>(matrix.col(0).head(4).asDiagonal(), + matrix.col(0).head(4).asDiagonal().toDenseMatrix()); + // initializer list + plain_storage<MatrixMax> storage; + storage.fill(); + MatrixMax* m = new (storage.bytes) MatrixMax({{1.0, 2.0}, {3.0, 4.0}}); + VERIFY_IS_EQUAL(m->rows(), 2); + VERIFY_IS_EQUAL(m->cols(), 2); + VERIFY_IS_EQUAL((*m)(1, 0), 3.0); + verify_unused_storage_untouched(*m); +} + +// The constructors above default-initialize the storage. Every coefficient of a scalar with a non-trivial default +// constructor must still be constructed before it is assigned: SafeScalar asserts on any read of a value that was not. +void fixed_max_safe_scalar_tests() { + using Scalar = SafeScalar<float>; + using MatrixMax = Matrix<Scalar, Dynamic, Dynamic, ColMajor, 8, 8>; + using VectorMax = Matrix<Scalar, Dynamic, 1, ColMajor, 16, 1>; + using Vector4 = Matrix<Scalar, 4, 1>; + MatrixMax source = MatrixMax::Random(8, 8); + // conversion from a DenseBase expression + MatrixMax dense = source.topLeftCorner(3, 5); + VERIFY_IS_CWISE_EQUAL(dense, source.topLeftCorner(3, 5)); + VectorMax vector = source.col(1).head(5); + VERIFY_IS_CWISE_EQUAL(vector, source.col(1).head(5)); + // conversion from an EigenBase expression + MatrixMax diagonal = source.col(0).head(4).asDiagonal(); + VERIFY_IS_CWISE_EQUAL(diagonal, source.col(0).head(4).asDiagonal().toDenseMatrix()); + // initializer list + MatrixMax list({{Scalar(1.f), Scalar(2.f)}, {Scalar(3.f), Scalar(4.f)}}); + VERIFY_IS_EQUAL(list.rows(), 2); + VERIFY_IS_EQUAL(list.cols(), 2); + VERIFY_IS_EQUAL(float(list(1, 0)), 3.f); + // four or more coefficients + Vector4 coefficients(Scalar(1.f), Scalar(2.f), Scalar(3.f), Scalar(4.f)); + VERIFY_IS_EQUAL(float(coefficients(3)), 4.f); +} + EIGEN_DECLARE_TEST(dense_storage) { dense_storage_tests<int>(); dense_storage_tests<float>(); dense_storage_tests<SafeScalar<float>>(); dense_storage_tests<MovableScalar<float>>(); dense_storage_tests<AnnoyingScalar>(); + fixed_max_storage_tests(); + fixed_max_safe_scalar_tests(); for (int i = 0; i < g_repeat; i++) { plaintype_tests<Matrix<float, 0, 0, ColMajor>>(); plaintype_tests<Matrix<float, Dynamic, Dynamic, ColMajor, 0, 0>>();
diff --git a/test/diagonalmatrices.cpp b/test/diagonalmatrices.cpp index 12e4e78..0ffdd5a 100644 --- a/test/diagonalmatrices.cpp +++ b/test/diagonalmatrices.cpp
@@ -132,12 +132,142 @@ VERIFY_IS_APPROX(sq_m3 = v1.asDiagonal() - v2.asDiagonal(), sq_m1 - sq_m2); VERIFY_IS_APPROX(sq_m3 = v1.asDiagonal() - 2 * v2.asDiagonal() + v1.asDiagonal(), sq_m1 - 2 * sq_m2 + sq_m1); + // transpose(), conjugate() and adjoint() are diagonal expressions: D^T = D and D^H = conj(D). + { + const SquareMatrixType dense1 = ldm1.toDenseMatrix(); + VERIFY_IS_EQUAL(SquareMatrixType(ldm1.transpose().toDenseMatrix()), SquareMatrixType(dense1.transpose())); + VERIFY_IS_EQUAL(SquareMatrixType(ldm1.conjugate().toDenseMatrix()), SquareMatrixType(dense1.conjugate())); + VERIFY_IS_EQUAL(SquareMatrixType(ldm1.adjoint().toDenseMatrix()), SquareMatrixType(dense1.adjoint())); + VERIFY_IS_EQUAL(VectorType((v1 + v2).asDiagonal().adjoint().diagonal()), VectorType((v1 + v2).conjugate())); + VERIFY_IS_APPROX(ldm1.adjoint() * m1, dense1.adjoint() * m1); + VERIFY_IS_APPROX(m1 * rdm1.adjoint(), m1 * rdm1.toDenseMatrix().adjoint()); + VERIFY_IS_APPROX(VectorType((ldm1.adjoint() * ldm2).diagonal()), + VectorType((dense1.adjoint() * ldm2.toDenseMatrix()).diagonal())); + internal::set_is_malloc_allowed(false); + VERIFY_IS_APPROX((ldm1.adjoint() * m1)(i, j), numext::conj(v1(i)) * m1(i, j)); + VERIFY_IS_APPROX((m1 * rdm1.transpose())(i, j), m1(i, j) * rv1(j)); + internal::set_is_malloc_allowed(true); + // For real scalars the conjugate is the diagonal itself, with no conjugation functor. + STATIC_CHECK( + (NumTraits<Scalar>::IsComplex || + internal::is_same<std::decay_t<decltype(ldm1.conjugate())>, DiagonalWrapper<const VectorType>>::value)); + } + // Zero and Identity LeftDiagonalMatrix zero = LeftDiagonalMatrix::Zero(rows); LeftDiagonalMatrix identity = LeftDiagonalMatrix::Identity(rows); VERIFY_IS_APPROX(identity.diagonal().sum(), Scalar(rows)); VERIFY_IS_APPROX(zero.diagonal().sum(), Scalar(0)); VERIFY_IS_APPROX((zero + 2 * LeftDiagonalMatrix::Identity(rows)).diagonal().sum(), Scalar(2 * rows)); + + // Lazy sums and differences with a dense matrix: the expression is a CwiseBinaryOp over the diagonal itself, + // so building it and reading one coefficient allocates nothing. + STATIC_CHECK((internal::is_same<decltype(sq_m1 + ldm1), + const CwiseBinaryOp<internal::scalar_sum_op<Scalar, Scalar>, const SquareMatrixType, + const LeftDiagonalMatrix>>::value)); + sq_m1 = random_for_arithmetic<SquareMatrixType>(rows, rows); + const SquareMatrixType ldm1Dense = ldm1.toDenseMatrix(); + internal::set_is_malloc_allowed(false); + VERIFY_IS_EQUAL((sq_m1 + ldm1)(i, i), sq_m1(i, i) + v1(i)); + VERIFY_IS_EQUAL((ldm1 - sq_m1)(i, i), v1(i) - sq_m1(i, i)); + const Index k = (i + 1) % rows; // off the diagonal unless rows == 1 + VERIFY_IS_EQUAL((sq_m1 + v1.asDiagonal())(i, k), sq_m1(i, k) + (i == k ? v1(i) : Scalar(0))); + internal::set_is_malloc_allowed(true); + VERIFY_IS_EQUAL(sq_m2 = sq_m1 + ldm1, SquareMatrixType(sq_m1 + ldm1Dense)); + VERIFY_IS_EQUAL(sq_m2 = ldm1 + sq_m1, SquareMatrixType(ldm1Dense + sq_m1)); + VERIFY_IS_EQUAL(sq_m2 = sq_m1 - ldm1, SquareMatrixType(sq_m1 - ldm1Dense)); + VERIFY_IS_EQUAL(sq_m2 = ldm1 - sq_m1, SquareMatrixType(ldm1Dense - sq_m1)); + VERIFY_IS_EQUAL(sq_m2 = sq_m1 + v1.asDiagonal(), SquareMatrixType(sq_m1 + ldm1Dense)); + VERIFY_IS_EQUAL(sq_m2 = (v1 + v2).asDiagonal() - sq_m1, + SquareMatrixType((v1 + v2).asDiagonal().toDenseMatrix() - sq_m1)); + VERIFY_IS_EQUAL(sq_m2 = sq_m1.transpose() + ldm1 + ldm2, + SquareMatrixType(sq_m1.transpose() + ldm1Dense + ldm2.toDenseMatrix())); + VERIFY_IS_APPROX((sq_m1 + ldm1) * v2, (sq_m1 + ldm1Dense) * v2); + VERIFY_IS_APPROX((sq_m1 + ldm1).sum(), (sq_m1 + ldm1Dense).sum()); + VERIFY_IS_APPROX((s1 * (sq_m1 - ldm1)).eval(), s1 * (sq_m1 - ldm1Dense)); + + // Direct assignment writes each column as vectorized segments of the dense operand around the diagonal + // coefficient: compound assignment, the negated forms, self-assignment of the dense operand, and a product + // operand on either side (the right one keeps the "xpr + product" rule). + sq_m2 = random_for_arithmetic<SquareMatrixType>(rows, rows); + sq_m3 = sq_m2; + sq_m2 += sq_m1 + ldm1; + sq_m3 += sq_m1 + ldm1Dense; + VERIFY_IS_EQUAL(sq_m2, sq_m3); + sq_m2 -= ldm1 - sq_m1; + sq_m3 -= ldm1Dense - sq_m1; + VERIFY_IS_EQUAL(sq_m2, sq_m3); + sq_m2 -= sq_m1 - ldm1; + sq_m3 -= sq_m1 - ldm1Dense; + VERIFY_IS_EQUAL(sq_m2, sq_m3); + sq_m2 = sq_m1; + sq_m2 = sq_m2 + ldm1; + VERIFY_IS_EQUAL(sq_m2, SquareMatrixType(sq_m1 + ldm1Dense)); + sq_m2 = sq_m1; + sq_m2 = ldm1 - sq_m2; + VERIFY_IS_EQUAL(sq_m2, SquareMatrixType(ldm1Dense - sq_m1)); + VERIFY_IS_APPROX(sq_m2 = sq_m1 * sq_m1 + ldm1, SquareMatrixType(sq_m1 * sq_m1 + ldm1Dense)); + VERIFY_IS_APPROX(sq_m2 = ldm1 + sq_m1 * sq_m1, SquareMatrixType(ldm1Dense + sq_m1 * sq_m1)); + VERIFY_IS_APPROX(sq_m2 = ldm1 - sq_m1 * sq_m1, SquareMatrixType(ldm1Dense - sq_m1 * sq_m1)); + VERIFY_IS_APPROX(sq_m2 = sq_m1 * sq_m1 - ldm1, SquareMatrixType(sq_m1 * sq_m1 - ldm1Dense)); + sq_m2 = sq_m1; + VERIFY_IS_APPROX(sq_m2 = sq_m2 * sq_m2 + ldm1, SquareMatrixType(sq_m1 * sq_m1 + ldm1Dense)); + + // The result is the coefficient-wise one, bit for bit. With dst scaled by 1024, dst + a drops low bits of a, + // so += and -= must add the exact sum at the diagonal; and a diagonal taken from the destination is read + // before it is overwritten. + const SquareMatrixType dst0 = Scalar(1024) * random_for_arithmetic<SquareMatrixType>(rows, rows); + const SquareMatrixType dst0Diagonal = dst0.diagonal().asDiagonal(); + sq_m2 = dst0; + sq_m3 = dst0; + sq_m2 += sq_m1 + ldm1; + sq_m3 += sq_m1 + ldm1Dense; + VERIFY_IS_EQUAL(sq_m2, sq_m3); + sq_m2 -= ldm1 - sq_m1; + sq_m3 -= ldm1Dense - sq_m1; + VERIFY_IS_EQUAL(sq_m2, sq_m3); + sq_m2 = dst0; + sq_m2 = sq_m1 + sq_m2.diagonal().asDiagonal(); + VERIFY_IS_EQUAL(sq_m2, SquareMatrixType(sq_m1 + dst0Diagonal)); + sq_m2 = dst0; + sq_m2 = sq_m2.diagonal().asDiagonal() - sq_m1; + VERIFY_IS_EQUAL(sq_m2, SquareMatrixType(dst0Diagonal - sq_m1)); + sq_m2 = dst0; + sq_m2 += sq_m1 - sq_m2.diagonal().asDiagonal(); + VERIFY_IS_EQUAL(sq_m2, SquareMatrixType(dst0 + (sq_m1 - dst0Diagonal))); +} + +// Operands for which a two-pass assignment (dense operand, then the diagonal) differs from the coefficient-wise sum. +template <int> +void dense_diagonal_sum_assignment_regressions() { + // dst + (a + d) is finite, (dst + a) + d is not. + Matrix<double, 1, 1> dst, a, d; + dst << 1e308; + a << 1e308; + d << -1e308; + dst += a + d.asDiagonal(); + VERIFY_IS_EQUAL(dst(0, 0), 1e308); + // d - a is defined for d = a = INT_MIN, -a is not (-fsanitize=undefined reports it). + Matrix<int, 1, 1> ai, r; + ai << NumTraits<int>::lowest(); + const Matrix<int, 1, 1> di = ai; + r = di.asDiagonal() - ai; + VERIFY_IS_EQUAL(r(0, 0), 0); + // dst - (a - d) is defined for dst = INT_MIN and a = d = 1, dst - a is not. + r << NumTraits<int>::lowest(); + ai << 1; + r -= ai - ai.asDiagonal(); + VERIFY_IS_EQUAL(r(0, 0), NumTraits<int>::lowest()); + // bool has no negation or subtraction, so a sum must not instantiate them. + Matrix<bool, 3, 3> b = Matrix<bool, 3, 3>::Zero(); + b(0, 1) = true; + const DiagonalMatrix<bool, 3> bd(true, false, true); + Matrix<bool, 3, 3> expected = b; + expected(0, 0) = expected(2, 2) = true; + Matrix<bool, 3, 3> bs = b + bd; + VERIFY((bs.array() == expected.array()).all()); + bs = bd + b; + VERIFY((bs.array() == expected.array()).all()); } template <typename MatrixType> @@ -606,6 +736,7 @@ CALL_SUBTEST_10(selfadjoint_diagonal_products<0>()); CALL_SUBTEST_10(selfadjoint_diagonal_products_block_path<0>()); CALL_SUBTEST_10(structured_diagonal_aliasing<0>()); + CALL_SUBTEST_10(dense_diagonal_sum_assignment_regressions<0>()); CALL_SUBTEST_11(diagonal_matrix_moves<float>()); CALL_SUBTEST_11(diagonal_matrix_moves<double>()); CALL_SUBTEST_11(diagonal_matrix_moves<int>());
diff --git a/test/eigensolver_selfadjoint.cpp b/test/eigensolver_selfadjoint.cpp index cab5f8d..3024d37 100644 --- a/test/eigensolver_selfadjoint.cpp +++ b/test/eigensolver_selfadjoint.cpp
@@ -416,6 +416,7 @@ } } +template <typename = void> void selfadjointeigensolver_subnormal_coefficients() { volatile float normalMinInput = (std::numeric_limits<float>::min)(); const double scale = 256.0 * double(normalMinInput); @@ -970,6 +971,7 @@ } #if defined(EIGEN_TEST_PART_20) || defined(EIGEN_TEST_PART_ALL) +template <typename = void> void selfadjoint_iterative_scaling_rounding() { Matrix4f matrix = Matrix4f::Zero(); matrix.diagonal() << numext::bit_cast<float>(numext::uint32_t(0x44123456)), @@ -1331,7 +1333,7 @@ #endif EIGEN_DECLARE_TEST(eigensolver_selfadjoint) { - CALL_SUBTEST_20(selfadjoint_iterative_scaling_rounding()); + CALL_SUBTEST_20(selfadjoint_iterative_scaling_rounding<>()); CALL_SUBTEST_20((selfadjoint_iterative_scaling_blocks<float, ColMajor>())); CALL_SUBTEST_20((selfadjoint_iterative_scaling_blocks<double, RowMajor>())); CALL_SUBTEST_20((selfadjoint_iterative_scaling_blocks<std::complex<float>, RowMajor>())); @@ -1361,7 +1363,7 @@ CALL_SUBTEST_5(generalizedselfadjointeigensolver_no_malloc<MatrixXcd>()); CALL_SUBTEST_5(selfadjointeigensolver_dense_deflation_scale_invariance<float>()); CALL_SUBTEST_5(selfadjointeigensolver_dense_deflation_scale_invariance<double>()); - CALL_SUBTEST_13(selfadjointeigensolver_subnormal_coefficients()); + CALL_SUBTEST_13(selfadjointeigensolver_subnormal_coefficients<>()); for (int i = 0; i < g_repeat; i++) { // trivial test for 1x1 matrices:
diff --git a/test/fp_control.h b/test/fp_control.h index a526152..af578d6 100644 --- a/test/fp_control.h +++ b/test/fp_control.h
@@ -5,6 +5,7 @@ #define EIGEN_TEST_FP_CONTROL_H #include <cfenv> +#include <cmath> #include <cstdint> #include <limits> @@ -32,6 +33,14 @@ #define EIGEN_TEST_HAS_X86_FTZ 0 #endif +// MXCSR.DAZ is reserved, and writing it faults, on the SSE2 processors that predate SSE3; every x86-64 processor has +// it. +#if EIGEN_TEST_HAS_X86_FTZ && (EIGEN_ARCH_x86_64 || defined(__SSE3__)) +#define EIGEN_TEST_HAS_X86_DAZ 1 +#else +#define EIGEN_TEST_HAS_X86_DAZ 0 +#endif + #if !EIGEN_TEST_DEVICE_COMPILE && defined(_MSC_VER) && (EIGEN_TEST_ARCH_ARM32 || EIGEN_ARCH_ARM64) #include <float.h> #include <intrin.h> @@ -48,6 +57,14 @@ namespace Eigen { +// Which subnormals a flush-to-zero mode replaces with zero: x86 controls results (MXCSR.FTZ) and inputs (MXCSR.DAZ) +// separately, while the Arm FZ and MIPS FS bits flush both. +enum class FlushToZeroMode { None = 0, Results = 1, Inputs = 2, Both = 3 }; + +inline bool flushToZeroModeIncludes(FlushToZeroMode mode, FlushToZeroMode part) { + return (static_cast<int>(mode) & static_cast<int>(part)) == static_cast<int>(part); +} + // C++14 has no standard FTZ control. <cfenv> preserves the standard state; // architecture-specific registers preserve and control FTZ. // @@ -60,59 +77,87 @@ // and IV-j (FCSR/MSACSR FS). class ScopedFlushToZero { public: - ScopedFlushToZero() { + // Selects at least the requested flushing and clears the rest: exactly the request where results and inputs have + // separate controls, both where one bit flushes both, gradual underflow for None. mode() is the control register + // read back afterwards, isSupported() whether it covers the request; the default request is FTZ alone on x86. + explicit ScopedFlushToZero(FlushToZeroMode requested = FlushToZeroMode::Results) { #if EIGEN_TEST_HAS_RUNTIME_FTZ environment_saved_ = std::fegetenv(&environment_) == 0; + const bool flush = requested != FlushToZeroMode::None; + EIGEN_UNUSED_VARIABLE(flush); #if EIGEN_TEST_HAS_X86_FTZ control_state_ = static_cast<std::uint32_t>(_mm_getcsr()); - _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); - active_ = true; + std::uint32_t wanted = 0; + if (flushToZeroModeIncludes(requested, FlushToZeroMode::Results)) wanted |= x86FlushToZeroMask(); +#if EIGEN_TEST_HAS_X86_DAZ + if (flushToZeroModeIncludes(requested, FlushToZeroMode::Inputs)) wanted |= x86DenormalsAreZeroMask(); +#endif + _mm_setcsr(static_cast<unsigned int>((control_state_ & ~x86FlushMask()) | wanted)); + restore_ = true; + const std::uint32_t actual = static_cast<std::uint32_t>(_mm_getcsr()) & x86FlushMask(); + mode_ = static_cast<FlushToZeroMode>(((actual & x86FlushToZeroMask()) != 0 ? 1 : 0) | + ((actual & x86DenormalsAreZeroMask()) != 0 ? 2 : 0)); #elif EIGEN_ARCH_ARM64 && defined(_MSC_VER) control_state_ = arm64ControlState(); - const std::uint64_t flush_to_zero = control_state_ | arm64FlushToZeroMask(); - _WriteStatusReg(0x5a20, static_cast<__int64>(flush_to_zero)); + const std::uint64_t wanted = flush ? arm64FlushToZeroMask() : std::uint64_t(0); + _WriteStatusReg(0x5a20, static_cast<__int64>((control_state_ & ~arm64FlushToZeroMask()) | wanted)); __isb(_ARM64_BARRIER_SY); - active_ = true; + restore_ = true; + mode_ = (arm64ControlState() & armFlushToZeroMask()) != 0 ? FlushToZeroMode::Both : FlushToZeroMode::None; #elif EIGEN_ARCH_ARM64 && (defined(__GNUC__) || defined(__clang__)) asm volatile("mrs %0, fpcr" : "=r"(control_state_)); - const std::uint64_t flush_to_zero = control_state_ | arm64FlushToZeroMask(); - asm volatile("msr fpcr, %0" : : "r"(flush_to_zero) : "memory"); + const std::uint64_t wanted = flush ? arm64FlushToZeroMask() : std::uint64_t(0); + const std::uint64_t fpcr = (control_state_ & ~arm64FlushToZeroMask()) | wanted; + asm volatile("msr fpcr, %0" : : "r"(fpcr) : "memory"); asm volatile("isb" : : : "memory"); - active_ = true; + restore_ = true; + std::uint64_t actual = 0; + asm volatile("mrs %0, fpcr" : "=r"(actual)); + mode_ = (actual & armFlushToZeroMask()) != 0 ? FlushToZeroMode::Both : FlushToZeroMode::None; #elif EIGEN_TEST_ARCH_ARM32 && defined(_MSC_VER) unsigned int current_control = 0; if (_controlfp_s(¤t_control, 0, 0) == 0) { control_state_ = current_control; - active_ = _controlfp_s(¤t_control, _DN_FLUSH, _MCW_DN) == 0; + restore_ = _controlfp_s(¤t_control, flush ? _DN_FLUSH : _DN_SAVE, _MCW_DN) == 0; + if (restore_ && _controlfp_s(¤t_control, 0, 0) == 0) { + mode_ = (current_control & _MCW_DN) == _DN_FLUSH ? FlushToZeroMode::Both : FlushToZeroMode::None; + } } #elif EIGEN_TEST_ARCH_ARM32 && (defined(__GNUC__) || defined(__clang__)) std::uint32_t fpscr = 0; asm volatile("vmrs %0, fpscr" : "=r"(fpscr)); control_state_ = fpscr; - fpscr |= armFlushToZeroMask(); + fpscr = (fpscr & ~armFlushToZeroMask()) | (flush ? armFlushToZeroMask() : std::uint32_t(0)); asm volatile("vmsr fpscr, %0" : : "r"(fpscr) : "memory"); asm volatile("isb" : : : "memory"); - active_ = true; + restore_ = true; + asm volatile("vmrs %0, fpscr" : "=r"(fpscr)); + mode_ = (fpscr & armFlushToZeroMask()) != 0 ? FlushToZeroMode::Both : FlushToZeroMode::None; #elif EIGEN_ARCH_MIPS && defined(__mips_hard_float) && (defined(__GNUC__) || defined(__clang__)) std::uint32_t fcsr = 0; asm volatile("cfc1 %0, $31" : "=r"(fcsr)); control_state_ = fcsr; - fcsr |= mipsFlushToZeroMask(); + fcsr = (fcsr & ~mipsFlushToZeroMask()) | (flush ? mipsFlushToZeroMask() : std::uint32_t(0)); asm volatile("ctc1 %0, $31" : : "r"(fcsr) : "memory"); #if defined(__mips_msa) asm volatile("cfcmsa %0, $1" : "=r"(vector_control_state_)); - const std::uint32_t msacsr = vector_control_state_ | mipsFlushToZeroMask(); + const std::uint32_t msacsr = + (vector_control_state_ & ~mipsFlushToZeroMask()) | (flush ? mipsFlushToZeroMask() : std::uint32_t(0)); asm volatile("ctcmsa $1, %0" : : "r"(msacsr) : "memory"); #endif - active_ = true; + restore_ = true; + asm volatile("cfc1 %0, $31" : "=r"(fcsr)); + mode_ = (fcsr & mipsFlushToZeroMask()) != 0 ? FlushToZeroMode::Both : FlushToZeroMode::None; #endif #endif + supported_ = requested == FlushToZeroMode::None ? mode_ == FlushToZeroMode::None + : (restore_ && flushToZeroModeIncludes(mode_, requested)); } ~ScopedFlushToZero() { #if EIGEN_TEST_HAS_RUNTIME_FTZ - if (!active_) return; + if (!restore_) return; // Restore both forms because the C++ floating-point environment is not // required to represent non-standard FTZ controls. @@ -145,19 +190,23 @@ #endif } - bool isSupported() const { return active_; } + // Whether the flushing read back from the control register covers the request. + bool isSupported() const { return supported_; } + + // The flushing the control register reads back after the constructor wrote it. + FlushToZeroMode mode() const { return mode_; } // Whether the hardware currently flushes subnormal inputs to zero, read from - // the registers the constructor writes. On x86 that is DAZ; the FTZ bit the - // constructor sets only flushes results. The Arm and MIPS controls flush - // both. A compiler told it may treat subnormals as zero leaves every bit - // clear, so this is a diagnostic, not a gate. Returns false where there is - // no runtime control to read. + // the registers the constructor writes. On x86 that is DAZ; the FTZ bit + // only flushes results. The Arm and MIPS controls flush both. A compiler + // told it may treat subnormals as zero leaves every bit clear, so this is a + // diagnostic, not a gate. Returns false where there is no runtime control + // to read. static bool hardwareFlushesSubnormalInputs() { #if !EIGEN_TEST_HAS_RUNTIME_FTZ return false; #elif EIGEN_TEST_HAS_X86_FTZ - return _MM_GET_DENORMALS_ZERO_MODE() == _MM_DENORMALS_ZERO_ON; + return (static_cast<std::uint32_t>(_mm_getcsr()) & x86DenormalsAreZeroMask()) != 0; #elif EIGEN_ARCH_ARM64 && defined(_MSC_VER) return (arm64ControlState() & arm64FlushToZeroMask()) != 0; #elif EIGEN_ARCH_ARM64 && (defined(__GNUC__) || defined(__clang__)) @@ -185,10 +234,16 @@ ScopedFlushToZero& operator=(const ScopedFlushToZero&) = delete; private: + // MXCSR bits 15 (FTZ) and 6 (DAZ). + static std::uint32_t x86FlushToZeroMask() { return std::uint32_t(1) << 15; } + static std::uint32_t x86DenormalsAreZeroMask() { return std::uint32_t(1) << 6; } + static std::uint32_t x86FlushMask() { return x86FlushToZeroMask() | x86DenormalsAreZeroMask(); } + + // FPSCR/FPCR bit 24 (FZ); the Arm64 mask adds bit 19 (FZ16) where half precision is in use. static std::uint32_t armFlushToZeroMask() { return std::uint32_t(1) << 24; } static std::uint64_t arm64FlushToZeroMask() { - std::uint64_t mask = std::uint64_t(1) << 24; + std::uint64_t mask = armFlushToZeroMask(); #if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) || defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) mask |= std::uint64_t(1) << 19; #endif @@ -203,7 +258,9 @@ // Declared only on the paths that use them: GCC 10 ignores `unused` on data // members, and clang warns on a private field no member function touches. - bool active_ = false; + bool restore_ = false; + bool supported_ = false; + FlushToZeroMode mode_ = FlushToZeroMode::None; #if EIGEN_TEST_HAS_RUNTIME_FTZ std::fenv_t environment_; bool environment_saved_ = false; @@ -214,6 +271,19 @@ #endif }; +// Calls func(mode) under gradual underflow and then under each flush-to-zero mode this host selects exactly, each +// read back from the control register: the four x86 combinations, None and Both on Arm and MIPS, and None alone +// where there is no runtime control. +template <typename Func> +void forEachFlushToZeroMode(const Func& func) { + for (const FlushToZeroMode requested : + {FlushToZeroMode::None, FlushToZeroMode::Results, FlushToZeroMode::Inputs, FlushToZeroMode::Both}) { + const ScopedFlushToZero scope(requested); + if (!scope.isSupported() || scope.mode() != requested) continue; + func(requested); + } +} + // min * 1/2 is subnormal, so the product is exactly zero precisely when results // are flushed to zero. The volatile operands keep the compiler from folding the // probe under IEEE semantics the run time does not use. @@ -224,6 +294,15 @@ return normal_min * one_half; } +// denorm_min * 2^digits is the normal 2^min_exponent, so the product is exactly zero precisely when subnormal inputs +// are flushed to zero (x86 DAZ, Arm FZ, MIPS FS); x86 FTZ alone leaves this normal result alone. +template <typename Scalar> +EIGEN_DONT_INLINE Scalar subnormalInputProbe() { + volatile Scalar denorm_min = std::numeric_limits<Scalar>::denorm_min(); + volatile Scalar scale = std::ldexp(Scalar(1), std::numeric_limits<Scalar>::digits); + return denorm_min * scale; +} + // Whether dividing a normal value by a subnormal divisor yields its IEEE 754 // quotient here. Two things defeat it: hardware that flushes subnormal inputs // (x86 DAZ, Arm FZ, MIPS FS) reads the divisor as zero, and a compiler @@ -269,6 +348,7 @@ } // namespace Eigen #undef EIGEN_TEST_HAS_RUNTIME_FTZ +#undef EIGEN_TEST_HAS_X86_DAZ #undef EIGEN_TEST_HAS_X86_FTZ #undef EIGEN_TEST_ARCH_ARM32 #undef EIGEN_TEST_DEVICE_COMPILE
diff --git a/test/fuzzy_matrix_checks.cpp b/test/fuzzy_matrix_checks.cpp new file mode 100644 index 0000000..05ce641 --- /dev/null +++ b/test/fuzzy_matrix_checks.cpp
@@ -0,0 +1,222 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include "main.h" +#include "fp_control.h" + +struct ZeroOnlyScalar { + double value; + ZeroOnlyScalar(double x = 0) : value(x) {} + friend ZeroOnlyScalar abs(const ZeroOnlyScalar& x) { return numext::abs(x.value); } + friend ZeroOnlyScalar operator*(const ZeroOnlyScalar& x, const ZeroOnlyScalar& y) { return x.value * y.value; } + friend bool operator<=(const ZeroOnlyScalar& x, const ZeroOnlyScalar& y) { return x.value <= y.value; } +}; + +struct ApproxOnlyScalar { + double value; + ApproxOnlyScalar(double x = 0) : value(x) {} +}; + +namespace Eigen { +template <> +struct NumTraits<ZeroOnlyScalar> : GenericNumTraits<ZeroOnlyScalar> {}; +template <> +struct NumTraits<ApproxOnlyScalar> : GenericNumTraits<ApproxOnlyScalar> {}; +namespace internal { +template <> +struct scalar_fuzzy_impl<ApproxOnlyScalar> { + static bool isApprox(const ApproxOnlyScalar& x, const ApproxOnlyScalar& y, const ApproxOnlyScalar& precision) { + return internal::isApprox(x.value, y.value, precision.value); + } +}; +} // namespace internal +} // namespace Eigen + +void check_fuzzy_custom_scalars() { + // Neither operation may instantiate the other scalar predicate, including in C++14. + Matrix<ZeroOnlyScalar, 2, 1> zero; + zero(0) = 0; + zero(1) = 0; + VERIFY(zero.isZero(ZeroOnlyScalar(1e-6))); + zero(1) = 1; + VERIFY(!zero.isZero(ZeroOnlyScalar(1e-6))); + Matrix<ZeroOnlyScalar, Dynamic, 1> dynamicZero = zero; + VERIFY(!dynamicZero.isZero(ZeroOnlyScalar(1e-6))); + dynamicZero.resize(0); + VERIFY(dynamicZero.isZero(ZeroOnlyScalar(1e-6))); + + Matrix<ApproxOnlyScalar, 2, 1> constant; + constant(0) = 0.5; + constant(1) = 0.5; + VERIFY(constant.isApproxToConstant(ApproxOnlyScalar(0.5), ApproxOnlyScalar(1e-6))); + constant(0) = 1; + VERIFY(!constant.isApproxToConstant(ApproxOnlyScalar(0.5), ApproxOnlyScalar(1e-6))); + Matrix<ApproxOnlyScalar, Dynamic, 1> dynamicConstant = constant; + VERIFY(!dynamicConstant.isApproxToConstant(ApproxOnlyScalar(0.5), ApproxOnlyScalar(1e-6))); + dynamicConstant.resize(0); + VERIFY(dynamicConstant.isApproxToConstant(ApproxOnlyScalar(0.5), ApproxOnlyScalar(1e-6))); +} + +template <typename Scalar, int Order> +void check_fuzzy_loops() { + using Mat = Matrix<Scalar, Dynamic, Dynamic, Order>; + const Scalar nan = NumTraits<Scalar>::quiet_NaN(); + for (Index rows : {0, 1, 3, 17, 31, 32, 33}) { + for (Index cols : {0, 1, 5, 19, 31, 32, 33}) { + Mat a = Mat::Zero(rows, cols); + for (int variant = 0; variant < 6; ++variant) { + if (variant == 1) a.setIdentity(); + if (variant == 2) a.setConstant(Scalar(0.5)); + if (variant == 3 && a.size()) a(a.rows() - 1, a.cols() - 1) = nan; + if (variant == 4) a.setConstant(nan); + if (variant == 5) a.setConstant(NumTraits<Scalar>::infinity()); + for (Scalar precision : {Scalar(0), Scalar(0.1), Scalar(-0.1), nan, NumTraits<Scalar>::infinity()}) { + bool zero = true, constant = true, upper = true, lower = true; + Scalar upperMax = Scalar(-1), lowerMax = Scalar(-1); + for (Index j = 0; j < cols; ++j) { + for (Index i = 0; i < rows; ++i) { + zero &= internal::isMuchSmallerThan(a(i, j), Scalar(1), precision); + constant &= internal::isApprox(a(i, j), Scalar(0.5), precision); + const Scalar value = numext::abs(a(i, j)); + if (i <= j && value > upperMax) upperMax = value; + if (i >= j && value > lowerMax) lowerMax = value; + } + } + for (Index j = 0; j < cols; ++j) + for (Index i = 0; i < rows; ++i) { + if (i > j && numext::abs(a(i, j)) > upperMax * precision) upper = false; + if (i < j && numext::abs(a(i, j)) > lowerMax * precision) lower = false; + } + VERIFY_IS_EQUAL(a.isZero(precision), zero); + VERIFY_IS_EQUAL(a.isApproxToConstant(Scalar(0.5), precision), constant); + VERIFY_IS_EQUAL(a.isUpperTriangular(precision), upper); + VERIFY_IS_EQUAL(a.isLowerTriangular(precision), lower); + } + } + } + } +} + +template <typename Scalar, int Order> +void check_fuzzy_boundaries() { + using Mat = Matrix<Scalar, Dynamic, Dynamic, Order>; + using Visitor = internal::fuzzy_constant_visitor<Scalar, false>; + STATIC_CHECK((internal::visit_impl<Mat, Visitor, true>::LinearAccess)); + STATIC_CHECK((!internal::visit_impl<Block<Mat>, Visitor, true>::LinearAccess)); + constexpr int packetSize = internal::packet_traits<Scalar>::size; + for (Index size : + {Index(1), Index(packetSize - 1), Index(packetSize), Index(packetSize + 1), Index(2 * packetSize + 1)}) { + // Unaligned linear access, including a packet crossing a column boundary. + std::vector<Scalar> data(3 * size + 1); + Map<Mat, Unaligned> matrix(data.data() + 1, 3, size); + for (Index k = 0; k < matrix.size(); ++k) { + matrix.setZero(); + VERIFY(matrix.isZero()); + matrix(k) = NumTraits<Scalar>::quiet_NaN(); + VERIFY(!matrix.isZero()); + matrix.setConstant(Scalar(0.5)); + VERIFY(matrix.isApproxToConstant(Scalar(0.5))); + matrix(k) = Scalar(1); + VERIFY(!matrix.isApproxToConstant(Scalar(0.5))); + } + } + for (Index inner : {Index(1), Index(packetSize), Index(packetSize + 1), Index(2 * packetSize + 1)}) { + Mat storage = Mat::Zero(inner + 2, inner + 2); + auto block = storage.block(1, 1, inner, inner); + for (Index k = 0; k < inner * inner; ++k) { + block.setZero(); + VERIFY(block.isZero()); + block(k / inner, k % inner) = Scalar(1); + VERIFY(!block.isZero()); + block.setConstant(Scalar(0.5)); + VERIFY(block.isApproxToConstant(Scalar(0.5))); + block(k / inner, k % inner) = Scalar(1); + VERIFY(!block.isApproxToConstant(Scalar(0.5))); + } + } + Matrix<Scalar, 3, 3, Order> fixed = Matrix<Scalar, 3, 3, Order>::Zero(); + VERIFY(fixed.isZero()); + fixed(2, 2) = Scalar(1); + VERIFY(!fixed.isZero()); + for (Index k = 0; k < fixed.size(); ++k) { + fixed.setZero(); + fixed(k) = Scalar(1); + VERIFY(!fixed.isZero()); + fixed.setConstant(Scalar(0.5)); + fixed(k) = Scalar(1); + VERIFY(!fixed.isApproxToConstant(Scalar(0.5))); + } + Matrix<Scalar, packetSize + 1, 1> packetTail; + for (Index k = 0; k < packetTail.size(); ++k) { + packetTail.setZero(); + packetTail(k) = Scalar(1); + VERIFY(!packetTail.isZero()); + packetTail.setConstant(Scalar(0.5)); + packetTail(k) = Scalar(1); + VERIFY(!packetTail.isApproxToConstant(Scalar(0.5))); + } + Scalar data[18] = {}; + Map<Matrix<Scalar, Dynamic, 1>, Unaligned, InnerStride<2>> strided(data, 9); + VERIFY(strided.isZero()); + strided(8) = Scalar(1); + VERIFY(!strided.isZero()); +} + +template <typename Scalar, int Order> +void check_fuzzy_subnormals() { + const ScopedFlushToZero gradual(FlushToZeroMode::None); + // Require gradual scalar arithmetic, even if the packet unit flushes (ARMv7 NEON). + if (subnormalInputProbe<Scalar>() == Scalar(0) || underflowProbe<Scalar>() == Scalar(0)) return; + const Scalar subnormal = (std::numeric_limits<Scalar>::min)() / Scalar(4); + using Mat = Matrix<Scalar, Dynamic, Dynamic, Order>; + using Visitor = internal::fuzzy_constant_visitor<Scalar, false>; + STATIC_CHECK( + (!EIGEN_ARCH_ARM || !std::is_same<Scalar, float>::value || !internal::functor_traits<Visitor>::PacketAccess)); + for (Index size : {1, 2, 4, 8, 9, 17}) { + Matrix<Scalar, Dynamic, 1> v = Matrix<Scalar, Dynamic, 1>::Zero(size); + for (Index k = 0; k < size; ++k) { + for (Scalar sign : {Scalar(1), Scalar(-1)}) { + v.setZero(); + v(k) = sign * subnormal; + VERIFY(!v.isZero(Scalar(0))); + VERIFY(!v.isConstant(Scalar(0))); + VERIFY(v.isZero(subnormal)); + VERIFY(!v.isZero(subnormal / Scalar(2))); + } + } + } + Mat c = Mat::Constant(4, 4, Scalar(2) * subnormal); + VERIFY(!c.isApproxToConstant(subnormal, Scalar(0))); + VERIFY(!c.isZero(subnormal)); + VERIFY(c.isApproxToConstant(Scalar(2) * subnormal, Scalar(0))); + VERIFY(c.isZero(Scalar(2) * subnormal)); + VERIFY(!c.block(0, 0, 3, 3).isZero(subnormal)); + VERIFY(!c.block(0, 0, 3, 3).isApproxToConstant(subnormal, Scalar(0))); + Matrix<Scalar, 3, 3, Order> fixed = c.template topLeftCorner<3, 3>(); + VERIFY(!fixed.isZero(subnormal)); + VERIFY(!fixed.isApproxToConstant(subnormal, Scalar(0))); + + // Normal operands can produce a subnormal difference in the approximate comparison. + const Scalar normal = (std::numeric_limits<Scalar>::min)(); + c.setConstant(normal + subnormal); + VERIFY(!c.isApproxToConstant(normal, Scalar(0))); + VERIFY(c.isApproxToConstant(normal + subnormal, Scalar(0))); +} + +EIGEN_DECLARE_TEST(fuzzy_matrix_checks) { + for (int repeat = 0; repeat < g_repeat; ++repeat) { + CALL_SUBTEST_1((check_fuzzy_boundaries<float, ColMajor>())); + CALL_SUBTEST_1((check_fuzzy_loops<float, ColMajor>())); + CALL_SUBTEST_1((check_fuzzy_subnormals<float, ColMajor>())); + CALL_SUBTEST_2((check_fuzzy_boundaries<double, ColMajor>())); + CALL_SUBTEST_2((check_fuzzy_loops<double, ColMajor>())); + CALL_SUBTEST_2((check_fuzzy_subnormals<double, ColMajor>())); + CALL_SUBTEST_3((check_fuzzy_boundaries<float, RowMajor>())); + CALL_SUBTEST_3((check_fuzzy_loops<float, RowMajor>())); + CALL_SUBTEST_3((check_fuzzy_subnormals<float, RowMajor>())); + CALL_SUBTEST_4((check_fuzzy_boundaries<double, RowMajor>())); + CALL_SUBTEST_4((check_fuzzy_loops<double, RowMajor>())); + CALL_SUBTEST_4((check_fuzzy_subnormals<double, RowMajor>())); + CALL_SUBTEST_5(check_fuzzy_custom_scalars()); + } +}
diff --git a/test/geo_misc.cpp b/test/geo_misc.cpp index ab0886d..9165594 100644 --- a/test/geo_misc.cpp +++ b/test/geo_misc.cpp
@@ -108,6 +108,40 @@ VERIFY_IS_APPROX(aa.toRotationMatrix(), aa2.toRotationMatrix()); } + // Near-180-degree rotation with negative dominant axis component: + // when sin(angle) is well above machine epsilon (e.g. angle = pi - 0.25 * sqrt(eps)), + // the axis sign is well-defined and must not flip. + { + Scalar delta = Scalar(0.25) * numext::sqrt(NumTraits<Scalar>::epsilon()); + Scalar a = Scalar(EIGEN_PI) - delta; + for (int d = 0; d < 3; ++d) { + Vector3 axis(Scalar(0.1), Scalar(-0.2), Scalar(0.15)); + axis(d) = Scalar(-1); + axis.normalize(); + AngleAxisx aa(a, axis); + Matrix3 m = aa.toRotationMatrix(); + AngleAxisx aa2; + aa2.fromRotationMatrix(m); + VERIFY_IS_APPROX(aa2.axis(), axis); + VERIFY_IS_APPROX(aa2.angle(), a); + } + } + + // Near-180-degree rotation extracted from a composed rotation matrix + // R_parent^T * (R_parent * R_true): rounding errors in R must not be + // amplified by 1 / sin(angle). + { + Vector3 r_true(Scalar(-3.134111), Scalar(-0.011655), Scalar(-0.003976)); + AngleAxisx aa_true(r_true.norm(), r_true.normalized()); + Matrix3 R_true = aa_true.toRotationMatrix(); + Vector3 r_parent(Scalar(1.946572), Scalar(-1.908926), Scalar(0.648041)); + Matrix3 R_parent = AngleAxisx(r_parent.norm(), r_parent.normalized()).toRotationMatrix(); + Matrix3 R_composed = R_parent.transpose() * (R_parent * R_true); + AngleAxisx aa_extracted(R_composed); + Vector3 r_extracted = aa_extracted.angle() * aa_extracted.axis(); + VERIFY((r_extracted - r_true).norm() < Scalar(16) * NumTraits<Scalar>::epsilon()); + } + // Accessors { Vector3 axis = Vector3::Random().normalized(); @@ -396,6 +430,27 @@ VERIFY(!(numext::isnan)(q.y())); VERIFY(!(numext::isnan)(q.z())); } + + // Pivot component must be rounded as 0.5 * sqrt(t) (single rounding, <= 0.5 ULP) + // rather than t * (0.5 / sqrt(t)) (which loses 1 ULP for t = 2 and 2 ULP for + // t = 3 in double, degrading downstream extraction on 90-deg and 60-deg rotations). + { + Matrix3 r90; + r90 << Scalar(0), Scalar(0), Scalar(1), Scalar(0), Scalar(1), Scalar(0), Scalar(-1), Scalar(0), Scalar(0); + Quaternionx q90(r90); + VERIFY_IS_EQUAL(q90.w(), Scalar(0.5) * numext::sqrt(Scalar(2))); + + Matrix3 r180_xy; + r180_xy << Scalar(0), Scalar(1), Scalar(0), Scalar(1), Scalar(0), Scalar(0), Scalar(0), Scalar(0), Scalar(-1); + Quaternionx q180(r180_xy); + VERIFY_IS_EQUAL(q180.x(), Scalar(0.5) * numext::sqrt(Scalar(2))); + + const Scalar s60 = numext::sqrt(Scalar(3)) * Scalar(0.5); + Matrix3 r60; + r60 << Scalar(0.5), -s60, Scalar(0), s60, Scalar(0.5), Scalar(0), Scalar(0), Scalar(0), Scalar(1); + Quaternionx q60(r60); + VERIFY_IS_EQUAL(q60.w(), Scalar(0.5) * numext::sqrt(Scalar(3))); + } } template <typename Scalar>
diff --git a/test/geo_transformations.cpp b/test/geo_transformations.cpp index 233b094..0ecb415 100644 --- a/test/geo_transformations.cpp +++ b/test/geo_transformations.cpp
@@ -71,6 +71,7 @@ }; } // namespace Eigen +template <typename = void> void custom_scalar_test() { { Transform<float, 3, Affine> tf = Transform<float, 3, Affine>::Identity(); @@ -947,7 +948,7 @@ CALL_SUBTEST_1((transformations<double, Affine, AutoAlign>())); CALL_SUBTEST_1((non_projective_only<double, Affine, AutoAlign>())); CALL_SUBTEST_1((transformations_computed_scaling_continuity<double, Affine, AutoAlign>())); - CALL_SUBTEST_1(custom_scalar_test()); + CALL_SUBTEST_1(custom_scalar_test<>()); CALL_SUBTEST_2((transformations<float, AffineCompact, AutoAlign>())); CALL_SUBTEST_2((non_projective_only<float, AffineCompact, AutoAlign>()));
diff --git a/test/gmres.cpp b/test/gmres.cpp index 5637ff9..be4d62d 100644 --- a/test/gmres.cpp +++ b/test/gmres.cpp
@@ -25,6 +25,7 @@ // CALL_SUBTEST( check_sparse_square_solving(gmres_colmajor_ssor) ); } +template <typename = void> void test_gmres_solve_with_guess_tolerance() { const SparseMatrix<double>::StorageIndex size = 20; SparseMatrix<double> matrix(size, size); @@ -58,6 +59,7 @@ VERIFY(gmres.error() < gmres.tolerance()); } +template <typename = void> void test_gmres_large_restart() { const SparseMatrix<double>::StorageIndex size = 4; SparseMatrix<double> matrix(size, size); @@ -82,6 +84,6 @@ EIGEN_DECLARE_TEST(gmres) { CALL_SUBTEST_1(test_gmres_T<double>()); CALL_SUBTEST_2(test_gmres_T<std::complex<double> >()); - CALL_SUBTEST_3(test_gmres_solve_with_guess_tolerance()); - CALL_SUBTEST_4(test_gmres_large_restart()); + CALL_SUBTEST_3(test_gmres_solve_with_guess_tolerance<>()); + CALL_SUBTEST_4(test_gmres_large_restart<>()); }
diff --git a/test/gpu_basic.cu b/test/gpu_basic.cu index 8f0a683..875f43c 100644 --- a/test/gpu_basic.cu +++ b/test/gpu_basic.cu
@@ -98,6 +98,37 @@ } }; +template <int Order> +struct mixed_outer_product { + EIGEN_DEVICE_FUNC void operator()(int i, const std::complex<float>* in, std::complex<float>* out) const { + using Mat = Eigen::Matrix<std::complex<float>, 4, 4, Order>; + const Eigen::Map<const Eigen::Vector4cf> complex(in + i); + const Eigen::Vector4f real = complex.real(); + Eigen::Map<Mat> result(out + i * 16); + if (Order == Eigen::RowMajor) { + result.noalias() = complex * real.transpose(); + result += complex.lazyProduct(real.transpose()); + } else { + result.noalias() = real * complex.transpose(); + result += real.lazyProduct(complex.transpose()); + } + } +}; + +template <int Order> +struct scaled_selfadjoint_diagonal_product { + EIGEN_DEVICE_FUNC void operator()(int i, const std::complex<float>* in, std::complex<float>* out) const { + using Scalar = std::complex<float>; + using Mat = Eigen::Matrix<Scalar, 3, 3, Order>; + const Eigen::Map<const Mat> matrix(in + i); + const Eigen::Vector3cf diagonal = Eigen::Vector3cf::Constant(Scalar(2, 1)); + const Scalar alpha(2, 3); + Eigen::Map<Mat> result(out + i * 9); + result = (alpha * matrix.template selfadjointView<Eigen::Lower>()) * diagonal.asDiagonal() + Mat::Zero(); + result += diagonal.asDiagonal() * (alpha * matrix.template selfadjointView<Eigen::Upper>()) + Mat::Zero(); + } +}; + struct make_householder_small_tail { EIGEN_DEVICE_FUNC void operator()(int i, const float* /*in*/, float* out) const { Eigen::Vector3f vector; @@ -380,6 +411,47 @@ } }; +template <typename T> +struct compound_product { + EIGEN_DEVICE_FUNC void operator()(int i, const typename T::Scalar* in, typename T::Scalar* out) const { + using Scalar = typename T::Scalar; + using ProdEval = + Eigen::internal::product_evaluator<Eigen::Product<T, T, Eigen::LazyProduct>, Eigen::CoeffBasedProductMode, + Eigen::DenseShape, Eigen::DenseShape>; + using AddOp = Eigen::internal::add_assign_op<Scalar, Scalar>; + static_assert(!Eigen::internal::product_packet_cascade_traits<ProdEval, T, AddOp>::Enable, + "product packet cascade must stay disabled when ActualPacketAccessBit == 0"); + constexpr int size = T::SizeAtCompileTime; + const T lhs(in + i); + const T rhs(in + i + 1); + const T addend(in + i + 2); + T accum = T::Zero(); + accum.noalias() += lhs * rhs; + accum.noalias() -= lhs * rhs; + accum += addend + lhs * rhs; +#if defined(EIGEN_GPU_COMPILE_PHASE) + using Packet = typename Eigen::internal::packet_traits<Scalar>::type; + if (size >= Eigen::internal::unpacket_traits<Packet>::size) { + const Packet zero = Eigen::internal::pzero(Packet()); + AddOp().template assignPacket<Eigen::Unaligned, Packet>(accum.data(), zero); + } +#endif + Eigen::Map<T>(out + i * size) = accum; + } +}; + +template <typename Scalar, int Options> +void test_compound_product() { + constexpr int n = 4; + Eigen::Array<Scalar, Eigen::Dynamic, 1> in(n * 512), out(n * 512); + in.setRandom(); + out.setZero(); + + run_and_compare_to_gpu(compound_product<Eigen::Matrix<Scalar, 3, 3, Options>>(), n, in, out); + run_and_compare_to_gpu(compound_product<Eigen::Matrix<Scalar, 5, 5, Options>>(), n, in, out); + run_and_compare_to_gpu(compound_product<Eigen::Matrix<Scalar, 9, 9, Options>>(), n, in, out); +} + template <typename T1, typename T2> struct diagonal { EIGEN_DEVICE_FUNC void operator()(int i, const typename T1::Scalar* in, typename T1::Scalar* out) const { @@ -730,12 +802,22 @@ CALL_SUBTEST(run_and_compare_to_gpu(prod_test<Matrix3f, Matrix3f>(), nthreads, in, out)); CALL_SUBTEST(run_and_compare_to_gpu(prod_test<Matrix4f, Vector4f>(), nthreads, in, out)); + CALL_SUBTEST((test_compound_product<float, ColMajor>())); + CALL_SUBTEST((test_compound_product<float, RowMajor>())); + CALL_SUBTEST((test_compound_product<double, ColMajor>())); + CALL_SUBTEST((test_compound_product<double, RowMajor>())); + CALL_SUBTEST((test_compound_product<Eigen::half, ColMajor>())); + CALL_SUBTEST((test_compound_product<Eigen::half, RowMajor>())); CALL_SUBTEST(run_and_compare_to_gpu(scaled_structured_product<RowMajor>(), nthreads, in, out)); CALL_SUBTEST(run_and_compare_to_gpu(scaled_structured_product<ColMajor>(), nthreads, in, out)); CALL_SUBTEST(run_and_compare_to_gpu(scaled_outer_product<RowMajor>(), nthreads, in, out)); CALL_SUBTEST(run_and_compare_to_gpu(scaled_outer_product<ColMajor>(), nthreads, in, out)); CALL_SUBTEST(run_and_compare_to_gpu(scaled_permutation_product<float>(), nthreads, in, out)); CALL_SUBTEST(run_and_compare_to_gpu(scaled_permutation_product<double>(), nthreads, in, out)); + CALL_SUBTEST(run_and_compare_to_gpu(mixed_outer_product<RowMajor>(), nthreads, cfin, cfout)); + CALL_SUBTEST(run_and_compare_to_gpu(mixed_outer_product<ColMajor>(), nthreads, cfin, cfout)); + CALL_SUBTEST(run_and_compare_to_gpu(scaled_selfadjoint_diagonal_product<RowMajor>(), nthreads, cfin, cfout)); + CALL_SUBTEST(run_and_compare_to_gpu(scaled_selfadjoint_diagonal_product<ColMajor>(), nthreads, cfin, cfout)); CALL_SUBTEST(run_and_compare_to_gpu(diagonal<Matrix3f, Vector3f>(), nthreads, in, out)); CALL_SUBTEST(run_and_compare_to_gpu(diagonal<Matrix4f, Vector4f>(), nthreads, in, out));
diff --git a/test/householder.cpp b/test/householder.cpp index 8e20328..896c089 100644 --- a/test/householder.cpp +++ b/test/householder.cpp
@@ -213,6 +213,43 @@ VERIFY_IS_APPROX(m3 * m5, m1); // test evaluating rhseq to a dense matrix, then applying } +// A row-vector operand gives a row-major row-vector result; a column-major one fails to instantiate. +template <typename Scalar> +void householder_row_vector_products(Index size) { + using MatrixType = Matrix<Scalar, Dynamic, Dynamic>; + using RowVectorType = Matrix<Scalar, 1, Dynamic>; + + const HouseholderQR<MatrixType> qr(MatrixType::Random(size, size)); + const MatrixType q = qr.householderQ(); + const RowVectorType row = RowVectorType::Random(size); + STATIC_CHECK((internal::is_same<decltype(row * qr.householderQ()), RowVectorType>::value)); + RowVectorType result = row * qr.householderQ(); + VERIFY_IS_APPROX(result, row * q); + result = row * qr.householderQ().adjoint(); + VERIFY_IS_APPROX(result, row * q.adjoint()); + + const HouseholderQR<MatrixType> qr1(MatrixType::Random(1, 1)); + const MatrixType q1 = qr1.householderQ(); + STATIC_CHECK((internal::is_same<decltype(qr1.householderQ() * row), RowVectorType>::value)); + result = qr1.householderQ() * row; + VERIFY_IS_APPROX(result, q1 * row); + + using Matrix3 = Matrix<Scalar, 3, 3>; + using RowVector3 = Matrix<Scalar, 1, 3>; + const HouseholderQR<Matrix3> qr3(Matrix3::Random()); + const RowVector3 row3 = RowVector3::Random(); + STATIC_CHECK((internal::is_same<decltype(row3 * qr3.householderQ()), RowVector3>::value)); + VERIFY_IS_APPROX(RowVector3(row3 * qr3.householderQ()), RowVector3(row3 * Matrix3(qr3.householderQ()))); + + // A single-column reflector set makes the blocked right-side tmp N x 1; it is instantiated although one + // reflector never takes that path at run time. + using VectorType = Matrix<Scalar, Dynamic, 1>; + const HouseholderQR<VectorType> qrv(VectorType::Random(size)); + const MatrixType qv = qrv.householderQ(); + const MatrixType m = MatrixType::Random(3, size); + VERIFY_IS_APPROX(MatrixType(m * qrv.householderQ()), m * qv); +} + template <typename MatrixType> void householder_update(const MatrixType& m) { // This test is covering the internal::householder_qr_inplace_update function. @@ -437,6 +474,7 @@ VERIFY((numext::isnan)(beta)); } +template <typename = void> void verify_custom_complex_small_tail() { typedef reordered_complex::Complex<float> Scalar; const float coefficient = 2e-16f; @@ -455,6 +493,7 @@ VERIFY_IS_EQUAL(numext::imag(essential[0]), 0.0f); } +template <typename = void> EIGEN_DONT_INLINE void verify_householder_flushed_tail() { volatile float normal_min_input = (std::numeric_limits<float>::min)(); const float normal_min = normal_min_input; @@ -473,6 +512,7 @@ verify_real_householder_result(vector, essential, tau, beta); } +template <typename = void> EIGEN_DONT_INLINE void verify_householder_dimension_scaled_tail() { // The dominant square is just above the length-independent threshold, while the many minor squares are normal // inputs whose individually subnormal products become significant in aggregate. @@ -496,6 +536,7 @@ VERIFY(std::abs(double(beta) + expected_norm) <= 8 * epsilon * expected_norm); } +template <typename = void> void householder_small_tail_layouts() { const float coefficient = 1e-20f; Vector4f column; @@ -537,6 +578,7 @@ VERIFY_IS_EQUAL(essential_storage[5], 23.0f); } +template <typename = void> void householder_small_tail() { { const float coefficient = 1e-20f; @@ -643,7 +685,7 @@ verify_low_precision_complex_zero_tail<bfloat16>(); verify_complex_nan_head<float>(); verify_complex_nan_head<double>(); - verify_custom_complex_small_tail(); + verify_custom_complex_small_tail<>(); { const float coefficient = 1e-20f; @@ -657,17 +699,17 @@ VERIFY(relative_residual <= 16 * double(NumTraits<float>::epsilon())); } - verify_householder_flushed_tail(); - verify_householder_dimension_scaled_tail(); + verify_householder_flushed_tail<>(); + verify_householder_dimension_scaled_tail<>(); { Eigen::ScopedFlushToZero flush_to_zero; if (flush_to_zero.isSupported()) { - verify_householder_flushed_tail(); - verify_householder_dimension_scaled_tail(); + verify_householder_flushed_tail<>(); + verify_householder_dimension_scaled_tail<>(); } } - householder_small_tail_layouts(); + householder_small_tail_layouts<>(); constexpr Index size = 65; const float coefficient = 1e-22f; @@ -697,6 +739,7 @@ verify_real_householder_result((vector / scale).eval(), essential, tau, RealScalar(beta / scale)); } +template <typename = void> void householder_large_components() { { // The head coefficient squares out of the float range; the tail coefficient does not. The direct @@ -950,6 +993,27 @@ VERIFY(a(0, 0) == Scalar(0)); VERIFY(a(1, 0) == -k); } + { + // A one-row block has no essential part, so H = 1 - tau = i: H * j = k on the left, j * H = -k on the right. + const Matrix<Scalar, 0, 1> none; + Scalar workspace[2]; + Mat row = Mat::Constant(1, 2, j); + row.applyHouseholderOnTheLeft(none, Scalar(1) - i, workspace); + VERIFY(row == Mat::Constant(1, 2, k)); + Mat column = Mat::Constant(2, 1, j); + column.applyHouseholderOnTheRight(none, Scalar(1) - i, workspace); + VERIFY(column == Mat::Constant(2, 1, -k)); + } + { + // A zero essential part keeps the product kernels out: a H = a - (a v) tau v^* with v = [1, 0...]^T, so + // tau = i turns a first column of j into j - j * i = j + k. + Mat a = Mat::Constant(2, Size, Scalar(0)); + a.col(0).setConstant(j); + const EssentialType essential = EssentialType::Constant(Scalar(0)); + Scalar workspace[2]; + a.applyHouseholderOnTheRight(essential, i, workspace); + VERIFY(a.col(0) == Mat::Constant(2, 1, j + k)); + } const Index cols = internal::random<Index>(1, 7); Mat a(Size, cols); for (Index c = 0; c < cols; ++c) @@ -967,6 +1031,199 @@ VERIFY(a == expected); } +// Whether applyHouseholderOnTheRight on Derived with this essential type selects the fused row loop. +template <typename Derived, typename EssentialPart> +constexpr bool householder_right_fused() { + return std::is_same<internal::householder_apply_right_impl<Derived, EssentialPart>, + internal::householder_apply_right_impl<Derived, EssentialPart, true>>::value; +} + +// a H = a - (a v) tau v^* with v = [1; essential], formed explicitly. +template <typename MatrixType, typename EssentialType> +MatrixType householder_right_reference(const MatrixType& a, const EssentialType& essential, + const typename MatrixType::Scalar& tau) { + using Scalar = typename MatrixType::Scalar; + Matrix<Scalar, Dynamic, 1> v(essential.size() + 1); + v << Scalar(1), essential; + return a - (a * v) * tau * v.adjoint(); +} + +// Applies [1; essential] and tau to columns j..j+K of a, through a fixed-width and a runtime-width block, with +// essential and tau either separate or stored in the block being updated. +template <int K, typename MatrixType> +void verify_householder_right_columns(const MatrixType& input, Index j) { + using Scalar = typename MatrixType::Scalar; + using RealScalar = typename NumTraits<Scalar>::Real; + using EssentialType = Matrix<Scalar, K, 1>; + using FixedBlock = Block<MatrixType, Dynamic, K + 1>; + using RuntimeBlock = Block<MatrixType, Dynamic, Dynamic>; + constexpr bool kFused = !MatrixType::IsRowMajor && internal::packet_traits<Scalar>::Vectorizable; + STATIC_CHECK((householder_right_fused<FixedBlock, EssentialType>() == kFused)); + STATIC_CHECK((householder_right_fused<RuntimeBlock, EssentialType>() == kFused)); + const Index rows = input.rows(); + Matrix<Scalar, Dynamic, 1> workspace(rows); + + const auto check = [&](const MatrixType& result, const EssentialType& essential, const Scalar& tau) { + MatrixType expected = input; + expected.middleCols(j, K + 1) = householder_right_reference(input.middleCols(j, K + 1).eval(), essential, tau); + // Each entry is a (K + 2)-term expression in a, essential and tau; the other columns must not change. + const RealScalar scale = RealScalar(1) + numext::abs(tau) * (RealScalar(1) + essential.squaredNorm()); + const RealScalar bound = RealScalar(16 * (K + 2)) * NumTraits<RealScalar>::epsilon() * scale * input.norm(); + VERIFY((result - expected).norm() <= bound); + }; + + Matrix<Scalar, K + 1, 1> x = Matrix<Scalar, K + 1, 1>::Random(); + EssentialType essential; + Scalar tau; + RealScalar beta; + x.makeHouseholder(essential, tau, beta); + MatrixType a = input; + a.template middleCols<K + 1>(j).applyHouseholderOnTheRight(essential, tau, workspace.data()); + check(a, essential, tau); + a = input; + a.middleCols(j, K + 1).applyHouseholderOnTheRight(essential, tau, workspace.data()); + check(a, essential, tau); + + // The fused loop copies essential and tau before its first store; here they are coefficients of row r of the + // block, so the reference uses the values read before the update. + if (kFused) { + const Index r = internal::random<Index>(0, rows - 1); + const EssentialType aliasedEssential = input.row(r).segment(j + 1, K).transpose(); + const Scalar aliasedTau = input(r, j); + a = input; + a.template middleCols<K + 1>(j).applyHouseholderOnTheRight(a.row(r).template segment<K>(j + 1).transpose(), a(r, j), + workspace.data()); + check(a, aliasedEssential, aliasedTau); + } +} + +// The two- and three-column right updates of the Francis QR step, over row counts that cover the packet loop and +// the scalar tail, with spare columns on both sides of the updated block. +template <typename Scalar, int StorageOrder> +void householder_right_fixed_columns() { + using Mat = Matrix<Scalar, Dynamic, Dynamic, StorageOrder>; + const Index maxRows = 4 * internal::packet_traits<Scalar>::size + 3; + for (Index rows = 1; rows <= maxRows; ++rows) { + const Mat input = Mat::Random(rows, 5); + verify_householder_right_columns<1>(input, internal::random<Index>(0, 3)); + verify_householder_right_columns<2>(input, internal::random<Index>(0, 2)); + } +} + +// Q times a diagonal, triangular, self-adjoint or permutation matrix, both orders, against the products with the +// dense Q. +template <typename Scalar> +void householder_structured_products(Index size) { + typedef Matrix<Scalar, Dynamic, Dynamic> MatrixType; + typedef Matrix<Scalar, Dynamic, 1> VectorType; + + const MatrixType a = MatrixType::Random(size, size); + const HouseholderQR<MatrixType> qr(a); + const MatrixType q = qr.householderQ(); + const DiagonalMatrix<Scalar, Dynamic> d(VectorType::Random(size)); + PermutationMatrix<Dynamic> p(size); + randomPermutationVector(p.indices(), size); + const MatrixType dDense = d.toDenseMatrix(); + const MatrixType pDense = p.toDenseMatrix().template cast<Scalar>(); + const MatrixType upperDense = a.template triangularView<Upper>(); + const MatrixType selfadjointDense = a.template selfadjointView<Lower>(); + + STATIC_CHECK((internal::is_same<decltype(qr.householderQ() * d), MatrixType>::value)); + STATIC_CHECK((internal::is_same<decltype(p * qr.householderQ()), MatrixType>::value)); + + MatrixType result = qr.householderQ() * d; + VERIFY_IS_APPROX(result, q * dDense); + result = d * qr.householderQ(); + VERIFY_IS_APPROX(result, dDense * q); + result = qr.householderQ() * a.template triangularView<Upper>(); + VERIFY_IS_APPROX(result, q * upperDense); + result = a.template triangularView<UnitLower>() * qr.householderQ(); + VERIFY_IS_APPROX(result, MatrixType(a.template triangularView<UnitLower>()) * q); + result = qr.householderQ() * a.template selfadjointView<Lower>(); + VERIFY_IS_APPROX(result, q * selfadjointDense); + result = a.template selfadjointView<Upper>() * qr.householderQ(); + VERIFY_IS_APPROX(result, MatrixType(a.template selfadjointView<Upper>()) * q); + result = qr.householderQ() * p; + VERIFY_IS_APPROX(result, q * pDense); + result = p * qr.householderQ(); + VERIFY_IS_APPROX(result, pDense * q); + result = qr.householderQ() * p.inverse(); + VERIFY_IS_APPROX(result, q * pDense.transpose()); + result = p.transpose() * qr.householderQ(); + VERIFY_IS_APPROX(result, pDense.transpose() * q); + result = qr.householderQ().adjoint() * d; + VERIFY_IS_APPROX(result, q.adjoint() * dDense); + result = qr.householderQ().adjoint() * p; + VERIFY_IS_APPROX(result, q.adjoint() * pDense); + + // Upper triangular operands on the left take the identity path of the dense product; the other triangular modes + // and the tall (non-square) case take the full path. + result = qr.householderQ() * qr.matrixQR().template triangularView<Upper>(); + VERIFY_IS_APPROX(result, a); + result = qr.householderQ() * a.template triangularView<StrictlyUpper>(); + VERIFY_IS_APPROX(result, q * MatrixType(a.template triangularView<StrictlyUpper>())); + result = qr.householderQ() * a.template triangularView<UnitUpper>(); + VERIFY_IS_APPROX(result, q * MatrixType(a.template triangularView<UnitUpper>())); + result = qr.householderQ() * a.template triangularView<Lower>(); + VERIFY_IS_APPROX(result, q * MatrixType(a.template triangularView<Lower>())); + const MatrixType tall = MatrixType::Random(2 * size, size); + const HouseholderQR<MatrixType> qrTall(tall); + result = qrTall.householderQ() * qrTall.matrixQR().template triangularView<Upper>(); + VERIFY_IS_APPROX(result, tall); + + // Real operands of a complex sequence promote to the sequence's scalar type. + using RealScalar = typename NumTraits<Scalar>::Real; + using RealMatrixType = Matrix<RealScalar, Dynamic, Dynamic>; + const DiagonalMatrix<RealScalar, Dynamic> dReal(Matrix<RealScalar, Dynamic, 1>::Random(size)); + const RealMatrixType aReal = RealMatrixType::Random(size, size); + STATIC_CHECK((internal::is_same<decltype(qr.householderQ() * dReal), MatrixType>::value)); + result = qr.householderQ() * dReal; + VERIFY_IS_APPROX(result, q * dReal.toDenseMatrix()); + result = dReal * qr.householderQ(); + VERIFY_IS_APPROX(result, dReal.toDenseMatrix() * q); + result = qr.householderQ() * aReal.template triangularView<Upper>(); + VERIFY_IS_APPROX(result, q * RealMatrixType(aReal.template triangularView<Upper>())); + result = aReal.template triangularView<Lower>() * qr.householderQ(); + VERIFY_IS_APPROX(result, RealMatrixType(aReal.template triangularView<Lower>()) * q); +} + +// D * H scales the rows of H from the left, D(i,i) * H(i,j), which only a noncommutative scalar tells apart from +// scaling them on the right. Integer components keep every operation exact. +template <typename = void> +void householder_structured_products_noncommutative() { + using Scalar = noncommutative_scalar::Quaternion; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using Vec = Matrix<Scalar, Dynamic, 1>; + { + // H = 1 - tau = j and D = i: D * H = i * j = k, where scaling on the right gives j * i = -k. + const Scalar i(0, 1), j(0, 0, 1), k(0, 0, 0, 1); + DiagonalMatrix<Scalar, Dynamic> d(1); + d.diagonal()(0) = i; + const Mat result = d * householderSequence(Mat::Constant(1, 1, Scalar(0)), Vec::Constant(1, Scalar(1) - j)); + VERIFY(result(0, 0) == k); + } + const auto randomScalar = [] { + return Scalar(internal::random<int>(-1, 1), internal::random<int>(-1, 1), internal::random<int>(-1, 1), + internal::random<int>(-1, 1)); + }; + const Index size = internal::random<Index>(1, 7); + Mat vectors(size, size); + Vec coeffs(size); + DiagonalMatrix<Scalar, Dynamic> d(size); + for (Index c = 0; c < size; ++c) { + for (Index r = 0; r < size; ++r) vectors(r, c) = randomScalar(); + coeffs(c) = randomScalar(); + d.diagonal()(c) = randomScalar(); + } + const HouseholderSequence<Mat, Vec> h(vectors, coeffs); + const Mat hDense = h; + Mat expected(size, size); + for (Index c = 0; c < size; ++c) + for (Index r = 0; r < size; ++r) expected(r, c) = d.diagonal()(r) * hDense(r, c); + const Mat result = d * h; + VERIFY(result == expected); +} + EIGEN_DECLARE_TEST(householder) { for (int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(householder(Matrix<double, 2, 2>())); @@ -977,6 +1234,10 @@ MatrixXd(internal::random<int>(1, EIGEN_TEST_MAX_SIZE), internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); CALL_SUBTEST_6(householder( MatrixXcf(internal::random<int>(1, EIGEN_TEST_MAX_SIZE), internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); + CALL_SUBTEST_5(householder_row_vector_products<double>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE))); + // Takes the blocked right-side application: length >= BlockSize (48) and rows >= 4 * BlockSize. + CALL_SUBTEST_5(householder_row_vector_products<double>(200)); + CALL_SUBTEST_6(householder_row_vector_products<std::complex<float>>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE))); CALL_SUBTEST_7(householder( MatrixXf(internal::random<int>(1, EIGEN_TEST_MAX_SIZE), internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); CALL_SUBTEST_8(householder(Matrix<double, 1, 1>())); @@ -989,8 +1250,8 @@ CALL_SUBTEST_10(householder_blocked_right_regression<double>()); CALL_SUBTEST_11(householder_blocked_right_regression<std::complex<double>>()); - CALL_SUBTEST_12(householder_small_tail()); - CALL_SUBTEST_13(householder_large_components()); + CALL_SUBTEST_12(householder_small_tail<>()); + CALL_SUBTEST_13(householder_large_components<>()); CALL_SUBTEST_14((householder_short_strided<float, 2, ColMajor>())); CALL_SUBTEST_14((householder_short_strided<float, 3, ColMajor>())); CALL_SUBTEST_15((householder_short_strided<double, 2, ColMajor>())); @@ -1009,5 +1270,17 @@ CALL_SUBTEST_21((householder_essential_expressions<std::complex<double>, RowMajor>())); CALL_SUBTEST_22(householder_noncommutative_scalar<2>()); CALL_SUBTEST_22(householder_noncommutative_scalar<3>()); + CALL_SUBTEST_23(householder_structured_products<float>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE))); + CALL_SUBTEST_23(householder_structured_products<double>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE))); + CALL_SUBTEST_23( + householder_structured_products<std::complex<double>>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE / 2))); + CALL_SUBTEST_23(householder_structured_products_noncommutative<>()); + CALL_SUBTEST_24((householder_right_fixed_columns<float, ColMajor>())); + CALL_SUBTEST_24((householder_right_fixed_columns<double, ColMajor>())); + CALL_SUBTEST_24((householder_right_fixed_columns<double, RowMajor>())); + CALL_SUBTEST_25((householder_right_fixed_columns<std::complex<float>, ColMajor>())); + CALL_SUBTEST_25((householder_right_fixed_columns<std::complex<double>, ColMajor>())); + CALL_SUBTEST_25((householder_right_fixed_columns<std::complex<double>, RowMajor>())); } + CALL_SUBTEST_23(householder_structured_products<double>(1)); }
diff --git a/test/indexed_view.cpp b/test/indexed_view.cpp index cb43d4c..3e40967 100644 --- a/test/indexed_view.cpp +++ b/test/indexed_view.cpp
@@ -950,6 +950,38 @@ VERIFY_IS_APPROX(row_result, row_expected); } +template <typename View> +void verify_strides_address_coeffs(const View& view) { + for (Index j = 0; j < view.cols(); ++j) { + for (Index i = 0; i < view.rows(); ++i) { + VERIFY_IS_EQUAL(view.data()[i * view.rowStride() + j * view.colStride()], view.coeff(i, j)); + } + } +} + +// A vector-shaped view takes its own storage order, which can differ from the nested one; its strides must still +// address its coefficients. +template <typename MatrixType> +void check_indexed_view_strides() { + const MatrixType A = MatrixType::Random(6, 8); + verify_strides_address_coeffs(A(2, seq(0, 7, 2))); + verify_strides_address_coeffs(A(2, seq(0, 7, fix<2>))); + verify_strides_address_coeffs(A(seq(0, 5, 2), 3)); + verify_strides_address_coeffs(A(seq(0, 5, fix<2>), 3)); + verify_strides_address_coeffs(A(seq(0, 5, 2), seq(1, 7, 3))); + + const Ref<const RowVectorXd, 0, InnerStride<>> row = A(2, seq(0, 7, 2)); + VERIFY_IS_EQUAL(row, RowVectorXd(A(2, seq(0, 7, 2)))); + const Ref<const VectorXd, 0, InnerStride<>> col = A(seq(0, 5, 2), 3); + VERIFY_IS_EQUAL(col, VectorXd(A(seq(0, 5, 2), 3))); + + // Matrix-vector products read a direct-access vector operand through data() and innerStride(). + const MatrixXd M = MatrixXd::Random(3, 4); + VERIFY_IS_APPROX(M * A(2, seq(0, 7, 2)).transpose(), M * RowVectorXd(A(2, seq(0, 7, 2))).transpose()); + const MatrixXd N = MatrixXd::Random(4, 3); + VERIFY_IS_APPROX(N * A(seq(0, 5, 2), 3), N * VectorXd(A(seq(0, 5, 2), 3))); +} + void check_aliasing() { Eigen::Vector<float, 5> z = {0.0f, 1.1f, 2.2f, 3.3f, 4.4f}; std::vector<int> left_indices = {0, 1, 3, 4}; @@ -967,6 +999,10 @@ CALL_SUBTEST_1(check_expression_indices()); CALL_SUBTEST_1(check_indexed_view_select()); CALL_SUBTEST_1(check_indexed_view_storage_order()); + CALL_SUBTEST_1(check_indexed_view_strides<MatrixXd>()); + CALL_SUBTEST_1((check_indexed_view_strides<Matrix<double, Dynamic, Dynamic, RowMajor>>())); + CALL_SUBTEST_1((check_indexed_view_strides<Matrix<double, 6, 8>>())); + CALL_SUBTEST_1((check_indexed_view_strides<Matrix<double, 6, 8, RowMajor>>())); CALL_SUBTEST_1(check_aliasing()); // static checks of some internals:
diff --git a/test/inplace_decomposition.cpp b/test/inplace_decomposition.cpp index 73b0c39..144937d 100644 --- a/test/inplace_decomposition.cpp +++ b/test/inplace_decomposition.cpp
@@ -389,6 +389,7 @@ } }; +template <typename = void> void inplace_plain_lower_triangle() { auto input = MatrixXd::NullaryExpr(3, 3, InplaceLowerOnly()); SelfAdjointEigenSolver<MatrixXd> mutableSolver(input); @@ -404,7 +405,7 @@ CALL_SUBTEST_4((inplace_fullpivlu_subspaces<Matrix<double, 3, 3, RowMajor | DontAlign>>())); CALL_SUBTEST_10((inplace_special_values<MatrixXd>(2))); CALL_SUBTEST_10((inplace_special_values<MatrixXd>(1))); - CALL_SUBTEST_11(inplace_plain_lower_triangle()); + CALL_SUBTEST_11(inplace_plain_lower_triangle<>()); CALL_SUBTEST_12((inplace_eigensolver<MatrixXd, EigenSolver>(128))); CALL_SUBTEST_12((inplace_eigensolver<MatrixXd, EigenSolver>(129))); CALL_SUBTEST_14((inplace_qz_inner_stride<double, RealQZ, ColMajor>()));
diff --git a/test/jacobi.cpp b/test/jacobi.cpp index 5316bab..70868aa 100644 --- a/test/jacobi.cpp +++ b/test/jacobi.cpp
@@ -12,6 +12,7 @@ #include "main.h" #include "fp_control.h" #include <Eigen/SVD> +#include <cfenv> template <typename MatrixType, typename JacobiScalar> void jacobi(const MatrixType& m = MatrixType()) { @@ -202,6 +203,37 @@ } template <typename Scalar> +EIGEN_DONT_INLINE void jacobi_makejacobi_bounded_ratio_impl(Scalar x, Scalar y, JacobiRotation<Scalar>& rotation) { + rotation.makeJacobi(x, y, Scalar(0)); +} + +template <typename Scalar> +void jacobi_makejacobi_bounded_ratio() { +#if defined(FE_OVERFLOW) && defined(FE_DIVBYZERO) && defined(FE_INVALID) + const Scalar small = (std::numeric_limits<Scalar>::min)(); + // The unused reciprocal either divides by zero, overflows, or overflows when squared. + for (const Scalar x : {Scalar(0), small, Scalar(1)}) { + for (const Scalar y : {small, Scalar(0.5)}) { + JacobiRotation<Scalar> rotation; + std::fenv_t environment; + if (std::feholdexcept(&environment) != 0) { + std::cout << "SKIP: makeJacobi exception check: feholdexcept failed.\n"; + return; + } + jacobi_makejacobi_bounded_ratio_impl(x, y, rotation); + const int exceptions = std::fetestexcept(FE_OVERFLOW | FE_DIVBYZERO | FE_INVALID); + const int restored = std::fesetenv(&environment); + VERIFY_IS_EQUAL(restored, 0); + VERIFY_IS_EQUAL(exceptions, 0); + const Scalar c = rotation.c(); + const Scalar s = rotation.s(); + VERIFY(numext::abs(c * c + s * s - Scalar(1)) <= Scalar(8) * NumTraits<Scalar>::epsilon()); + } + } +#endif +} + +template <typename Scalar> void jacobi_makejacobi_large_tau() { using std::abs; using std::sqrt; @@ -309,6 +341,8 @@ CALL_SUBTEST_7((jacobi_makegivens_safe_scaling<long double>())); CALL_SUBTEST_7((jacobi_makejacobi_large_tau<float>())); CALL_SUBTEST_7((jacobi_makejacobi_large_tau<double>())); + CALL_SUBTEST_7((jacobi_makejacobi_bounded_ratio<float>())); + CALL_SUBTEST_7((jacobi_makejacobi_bounded_ratio<double>())); CALL_SUBTEST_7((jacobi_makejacobi_extreme_phase<float>())); CALL_SUBTEST_7((jacobi_makejacobi_extreme_phase<double>())); CALL_SUBTEST_7((jacobi_makejacobi_ratio_boundaries<float>()));
diff --git a/test/jacobisvd.cpp b/test/jacobisvd.cpp index fd1b02a..faaa0d5 100644 --- a/test/jacobisvd.cpp +++ b/test/jacobisvd.cpp
@@ -39,9 +39,12 @@ template <typename MatrixType> void jacobisvd_thin_full_options(const MatrixType& input = MatrixType()) { svd_thin_full_option_checks<MatrixType, 0>(input); + svd_thin_full_option_checks<MatrixType, PreconditionSquareMatrix>(input); svd_thin_full_option_checks<MatrixType, HouseholderQRPreconditioner>(input); - svd_option_checks_full_only<MatrixType, FullPivHouseholderQRPreconditioner>( - input); // FullPiv only used when computing full unitaries + svd_thin_full_option_checks<MatrixType, HouseholderQRPreconditioner | PreconditionSquareMatrix>(input); + // FullPiv only used when computing full unitaries + svd_option_checks_full_only<MatrixType, FullPivHouseholderQRPreconditioner>(input); + svd_option_checks_full_only<MatrixType, FullPivHouseholderQRPreconditioner | PreconditionSquareMatrix>(input); } template <typename MatrixType> @@ -100,6 +103,7 @@ } // namespace Foo // regression test for a very strange MSVC issue for which simply // including SVDBase.h messes up with std::max and custom scalar type +template <typename = void> void msvc_workaround() { const Foo::Bar a; const Foo::Bar b; @@ -107,6 +111,7 @@ EIGEN_UNUSED_VARIABLE(c); } +template <typename = void> void jacobisvd_mixed_option_enum_regression() { using NoQrFullSVD = JacobiSVD<MatrixXd, NoQRPreconditioner | ComputeFullU | ComputeFullV>; using ReversedMixedSVD = JacobiSVD<MatrixXd, ComputeThinU | HouseholderQRPreconditioner | ComputeFullV>; @@ -124,6 +129,26 @@ STATIC_CHECK(((int(ReversedMixedSVD::Options) & ComputeThinV) == 0)); } +EIGEN_DIAGNOSTICS(push) +EIGEN_DISABLE_DEPRECATED_WARNING +// Thin unitaries requested through the deprecated runtime options must be rejected with FullPiv, as they are +// statically through Options. +template <int Options> +void jacobisvd_fullpiv_runtime_thin_asserts(Index rows, Index cols) { + using SVD = JacobiSVD<MatrixXd, FullPivHouseholderQRPreconditioner | Options>; + const MatrixXd m = MatrixXd::Random(rows, cols); + for (unsigned int thin : {unsigned(ComputeThinU), unsigned(ComputeThinV), unsigned(ComputeThinU | ComputeThinV)}) { + VERIFY_RAISES_ASSERT(SVD(m, thin)); + VERIFY_RAISES_ASSERT(SVD(rows, cols, thin)); + SVD svd; + VERIFY_RAISES_ASSERT(svd.compute(m, thin)); + } + SVD full(m, ComputeFullU | ComputeFullV); + svd_check_full(m, full); +} +EIGEN_DIAGNOSTICS(pop) + +template <typename = void> void jacobisvd_large_tau_regression() { Matrix3f m; m << 3.7855173218304116745e-07f, 0.0f, 500.0f, -4.9999995231628417969f, -0.0f, -1.9106853686029490191e-12f, @@ -148,6 +173,99 @@ svd_check_full(m, svd); } +// An all-subnormal matrix, which a SIMD unit that flushes subnormal inputs (ARMv7 NEON, Arm FZ, DAZ) reads as zero in +// the maxCoeff() selecting the scale: recovering the maximum from the representation and scaling through integer +// significands keeps the decomposition, and the singular values come back through the subnormal range exactly. +template <typename MatrixType> +void jacobisvd_flushed_subnormal_matrix(Index rows, Index cols) { + using RealScalar = typename MatrixType::RealScalar; + const MatrixType m = svd_subnormal_fixture<MatrixType>(rows, cols); + const int k = 60; + const MatrixType ms = m.unaryExpr(internal::scale_by_exponent_op<RealScalar>(k)); + const JacobiSVD<MatrixType, ComputeFullU | ComputeFullV> scaled(ms); + forEachFlushToZeroMode([&](FlushToZeroMode) { + const JacobiSVD<MatrixType, ComputeFullU | ComputeFullV> svd(m); + svd_check_flushed_subnormal(svd, scaled, m, k); + }); +} + +// The Jacobi sweep leaves a diagonal matrix untouched, so with distinct magnitudes |d_i| the singular values are +// those magnitudes bit for bit, descending, U the permutation carrying the signs of d and V the permutation, in every +// flush-to-zero mode. Signed subnormal entries with and without a normal one reach the sort of a subnormal tail, the +// extraction of magnitude and sign, and the nonzero count in the subnormal range. +template <typename RealScalar> +void jacobisvd_subnormal_diagonal(const Matrix<RealScalar, Dynamic, 1>& diagonal) { + using Binary = internal::binary_floating_point_traits<RealScalar>; + using Bits = typename Binary::Bits; + using MatrixType = Matrix<RealScalar, Dynamic, Dynamic>; + const Index n = diagonal.size(); + const MatrixType m = diagonal.asDiagonal(); + std::vector<Bits> magnitudes; + std::vector<Index> order; + Index nonzero = 0; + for (Index i = 0; i < n; ++i) { + magnitudes.push_back(Binary::magnitude(diagonal(i))); + order.push_back(i); + if (magnitudes.back() != 0) ++nonzero; + } + std::stable_sort(order.begin(), order.end(), [&](Index a, Index b) { return magnitudes[a] > magnitudes[b]; }); + for (Index j = 1; j < nonzero; ++j) VERIFY(magnitudes[order[j - 1]] != magnitudes[order[j]]); + + const auto checkValues = [&](const auto& svd) { + VERIFY_IS_EQUAL(svd.info(), Success); + VERIFY_IS_EQUAL(svd.nonzeroSingularValues(), nonzero); + for (Index j = 0; j < n; ++j) VERIFY_IS_EQUAL(Binary::bits(svd.singularValues()(j)), magnitudes[order[j]]); + }; + forEachFlushToZeroMode([&](FlushToZeroMode) { + const JacobiSVD<MatrixType> valuesOnly(m); + checkValues(valuesOnly); + const JacobiSVD<MatrixType, ComputeFullU | ComputeFullV> full(m); + checkValues(full); + VERIFY_IS_UNITARY(full.matrixU()); + VERIFY_IS_UNITARY(full.matrixV()); + for (Index j = 0; j < nonzero; ++j) { + const Index p = order[j]; + const bool negative = (Binary::bits(diagonal(p)) & Binary::kSignBit) != 0; + VERIFY_IS_EQUAL(full.matrixU().col(j).cwiseAbs().sum(), RealScalar(1)); + VERIFY_IS_EQUAL(full.matrixU()(p, j), negative ? RealScalar(-1) : RealScalar(1)); + VERIFY_IS_EQUAL(full.matrixV().col(j).cwiseAbs().sum(), RealScalar(1)); + VERIFY_IS_EQUAL(full.matrixV()(p, j), RealScalar(1)); + } + }); +} + +template <typename RealScalar> +void jacobisvd_subnormal_diagonals() { + using Binary = internal::binary_floating_point_traits<RealScalar>; + using Bits = typename Binary::Bits; + using VectorType = Matrix<RealScalar, Dynamic, 1>; + constexpr int digits = std::numeric_limits<RealScalar>::digits; + const auto subnormal = [](Bits significand, bool negative) { + return numext::bit_cast<RealScalar>((negative ? Binary::kSignBit : Bits(0)) | significand); + }; + const RealScalar denormMin = subnormal(Bits(1), false); + const RealScalar small = subnormal(Bits(1) << (digits - 3), true); + const RealScalar large = subnormal(Bits(1) << (digits - 2), false); + const RealScalar top = subnormal(Binary::kFractionMask, true); + // Signed subnormal entries only, with zeros and a signed zero in the tail. + VectorType allSubnormal(7); + allSubnormal << small, RealScalar(0), top, denormMin, large, subnormal(Bits(3), true), -RealScalar(0); + jacobisvd_subnormal_diagonal(allSubnormal); + // A normal entry in [1, 2): the identity scaling, and the subnormal tail sorted from the representation. + VectorType normalLeading(6); + normalLeading << RealScalar(1), RealScalar(0), small, large, RealScalar(-1.5), denormMin; + jacobisvd_subnormal_diagonal(normalLeading); + VectorType tail(2); + tail << RealScalar(1.5), denormMin; + jacobisvd_subnormal_diagonal(tail); + // A normal entry below min / eps: the integer scaling into and out of the subnormal range. + VectorType belowRecovery(5); + belowRecovery << subnormal(Bits(5) << (digits - 6), false), RealScalar(0), + -numext::ldexp(RealScalar(1), internal::safe_scaling<RealScalar>::subnormal_recovery_exponent() - 20), small, top; + jacobisvd_subnormal_diagonal(belowRecovery); +} + +template <typename = void> void jacobisvd_power_of_two_scaling() { // Reciprocal scaling rounds the smaller singular value up by one ULP. Matrix2f matrix = Matrix2f::Zero(); @@ -187,18 +305,7 @@ const JacobiSVD<Matrix<std::complex<float>, 2, 3>> tinyWideSvd(tinyWide); VERIFY((tinyWideSvd.singularValues() / inputScale - normalizedSvd.singularValues()).norm() <= tolerance); }; - checkSubnormalScaling(); - { - ScopedFlushToZero flushToZero; - checkSubnormalScaling(); - } - - volatile float denormMinInput = std::numeric_limits<float>::denorm_min(); - const float denormMin = denormMinInput; - if (!(denormMin > 0.0f)) return; - matrix.diagonal() << 1.5f, denormMin; - const JacobiSVD<Matrix2f> tailSvd(matrix); - VERIFY_IS_EQUAL(tailSvd.singularValues()(1), denormMin); + forEachFlushToZeroMode([&](FlushToZeroMode) { checkSubnormalScaling(); }); } EIGEN_DECLARE_TEST(jacobisvd) { @@ -210,8 +317,8 @@ CALL_SUBTEST_1((jacobisvd_verify_inputs<Matrix4d>())); CALL_SUBTEST_2((jacobisvd_verify_inputs(Matrix<float, 5, Dynamic>(5, 6)))); CALL_SUBTEST_3((jacobisvd_verify_inputs<Matrix<std::complex<double>, 7, 5>>())); - CALL_SUBTEST_4((jacobisvd_mixed_option_enum_regression())); - CALL_SUBTEST_4((jacobisvd_large_tau_regression())); + CALL_SUBTEST_4((jacobisvd_mixed_option_enum_regression<>())); + CALL_SUBTEST_4((jacobisvd_large_tau_regression<>())); CALL_SUBTEST_11((jacobisvd_thin_full_options<Matrix2cd>())); CALL_SUBTEST_12((jacobisvd_thin_full_options<Matrix2d>())); @@ -235,6 +342,22 @@ CALL_SUBTEST_33((jacobisvd_thin_full_options<Matrix<double, 5, 7, RowMajor>>())); CALL_SUBTEST_35((jacobisvd_thin_full_options<Matrix<double, 7, 5, RowMajor>>())); + CALL_SUBTEST_62((svd_precondition_square_matrix_accuracy<Matrix2f>(2))); + CALL_SUBTEST_62((svd_precondition_square_matrix_accuracy<Matrix4d>(4))); + CALL_SUBTEST_62((svd_precondition_square_matrix_accuracy<MatrixXf>(12))); + CALL_SUBTEST_62((svd_precondition_square_matrix_accuracy<MatrixXd>(12))); + CALL_SUBTEST_62((svd_precondition_square_matrix_accuracy<Matrix<double, Dynamic, Dynamic, RowMajor>>(12))); + CALL_SUBTEST_63((svd_precondition_square_matrix_accuracy<Matrix2cd>(2))); + CALL_SUBTEST_63((svd_precondition_square_matrix_accuracy<Matrix4cf>(4))); + CALL_SUBTEST_63((svd_precondition_square_matrix_accuracy<MatrixXcf>(12))); + CALL_SUBTEST_63((svd_precondition_square_matrix_accuracy<MatrixXcd>(12))); + CALL_SUBTEST_64((svd_precondition_square_matrix_runtime_options<MatrixXf>(12))); + CALL_SUBTEST_64((svd_precondition_square_matrix_runtime_options<MatrixXcd>(r))); + CALL_SUBTEST_64((jacobisvd_fullpiv_runtime_thin_asserts<0>(8, 5))); + CALL_SUBTEST_64((jacobisvd_fullpiv_runtime_thin_asserts<PreconditionSquareMatrix>(6, 6))); + CALL_SUBTEST_64((jacobisvd_thin_full_options<Matrix<double, Dynamic, Dynamic, RowMajor>>( + Matrix<double, Dynamic, Dynamic, RowMajor>(8, 8)))); + MatrixXcd noQRTest = MatrixXcd(r, r); CALL_SUBTEST_37((svd_thin_full_option_checks<MatrixXcd, NoQRPreconditioner>(noQRTest))); @@ -279,12 +402,21 @@ CALL_SUBTEST_55(svd_preallocate<void>()); CALL_SUBTEST_56(svd_underoverflow<void>()); - CALL_SUBTEST_56(jacobisvd_power_of_two_scaling()); + CALL_SUBTEST_56(jacobisvd_power_of_two_scaling<>()); + CALL_SUBTEST_56((jacobisvd_flushed_subnormal_matrix<MatrixXf>(5, 4))); + CALL_SUBTEST_56((jacobisvd_flushed_subnormal_matrix<MatrixXd>(4, 5))); + CALL_SUBTEST_56((jacobisvd_flushed_subnormal_matrix<MatrixXcf>(4, 4))); + CALL_SUBTEST_56((jacobisvd_flushed_subnormal_matrix<MatrixXcd>(3, 5))); + CALL_SUBTEST_56(jacobisvd_subnormal_diagonals<float>()); + CALL_SUBTEST_56(jacobisvd_subnormal_diagonals<double>()); + CALL_SUBTEST_56((svd_zero_matrix_solvers<Matrix3d>(3, 3))); + CALL_SUBTEST_56((svd_zero_matrix_solvers<Matrix<float, 2, 5>>(2, 5))); + CALL_SUBTEST_56((svd_zero_matrix_solvers<MatrixXcf>(5, 3))); // Check that the TriangularBase constructor works CALL_SUBTEST_57((svd_triangular_matrix<Matrix3d>())); CALL_SUBTEST_58((svd_triangular_matrix<Matrix4f>())); CALL_SUBTEST_59((svd_triangular_matrix<Matrix<double, 10, 10>>())); - msvc_workaround(); + msvc_workaround<>(); }
diff --git a/test/lu.cpp b/test/lu.cpp index d837e3e..1947ce9 100644 --- a/test/lu.cpp +++ b/test/lu.cpp
@@ -437,6 +437,95 @@ } } +// Fixed sizes up to 12 take the compile-time unrolled elimination; 13-16 take the runtime loop at a fixed size, and a +// dynamic matrix of the same size takes the runtime loop at a runtime size. Both must choose the same pivots and report +// the same first exactly-zero pivot. +template <typename Scalar, int N, int Options> +void lu_partial_piv_fixed() { + using Fixed = Matrix<Scalar, N, N, Options>; + using Dyn = Matrix<Scalar, Dynamic, Dynamic, Options>; + using Vector = Matrix<Scalar, N, 1>; + using RealScalar = typename NumTraits<Scalar>::Real; + using Pivots = Matrix<int, Dynamic, 1>; + constexpr int Order = (Options & RowMajor) ? RowMajor : ColMajor; + const RealScalar eps = NumTraits<RealScalar>::epsilon(); + + // Returns the index of the first exactly-zero pivot, which PartialPivLU discards, after checking that the fixed and + // the dynamic kernel agree on it, on the pivots, and on the parity of the transpositions. + auto first_zero_pivot = [](const Fixed& a) { + Fixed fixed_lu = a; + Dyn dyn_lu = a; + Matrix<int, N, 1> fixed_tr; + Pivots dyn_tr(N); + int fixed_nb = -1, dyn_nb = -1; + const Index fixed_zero = internal::partial_lu_impl<Scalar, Order, int, N>::blocked_lu( + N, N, fixed_lu.data(), fixed_lu.outerStride(), fixed_tr.data(), fixed_nb); + const Index dyn_zero = internal::partial_lu_impl<Scalar, Order, int, Dynamic>::blocked_lu( + N, N, dyn_lu.data(), dyn_lu.outerStride(), dyn_tr.data(), dyn_nb); + VERIFY_IS_EQUAL(fixed_zero, dyn_zero); + VERIFY_IS_EQUAL(Pivots(fixed_tr), dyn_tr); + VERIFY_IS_EQUAL(fixed_nb, dyn_nb); + return fixed_zero; + }; + + // P A = L U holds to the backward error bound |P A - L U| <= gamma_n |L| |U|, gamma_n ~ n u = n eps / 2; the factor + // 4 n eps leaves headroom for the Frobenius norm. The same bound holds for the residual of solve() relative to |x|. + auto check = [&](const Fixed& a) { + PartialPivLU<Fixed> lu(a); + PartialPivLU<Dyn> dyn(a); + VERIFY_IS_EQUAL(Pivots(lu.permutationP().indices()), Pivots(dyn.permutationP().indices())); + const Fixed lower = lu.matrixLU().template triangularView<UnitLower>(); + const Fixed upper = lu.matrixLU().template triangularView<Upper>(); + const RealScalar bound = RealScalar(4 * N) * eps * (lower.cwiseAbs() * upper.cwiseAbs()).norm(); + VERIFY((numext::isfinite)(bound)); + VERIFY((lu.permutationP() * a - lower * upper).norm() <= bound); + VERIFY((lu.matrixLU() - dyn.matrixLU()).norm() <= bound); + return bound; + }; + + // Nonsingular random matrices: most steps swap rows, and every step runs a full rank-1 update. + for (int repeat = 0; repeat < 4; ++repeat) { + const Fixed a = Fixed::Random(); + const RealScalar bound = check(a); + VERIFY_IS_EQUAL(first_zero_pivot(a), Index(-1)); + const PartialPivLU<Fixed> lu(a); + VERIFY_IS_APPROX(lu.determinant(), PartialPivLU<Dyn>(a).determinant()); + const Vector b = Vector::Random(); + const Vector x = lu.solve(b); + VERIFY((a * x - b).norm() <= bound * x.norm()); + } + + // A row-permuted upper triangular matrix with diagonal 1, ..., N: partial pivoting recovers U exactly with L = I, so + // the determinant is exactly sign(perm) N!, which checks the transposition count. N! is exact in float up to N = 13. + { + PermutationMatrix<N, N, int> perm; + perm.setIdentity(); + for (int i = N - 1; i > 0; --i) std::swap(perm.indices()[i], perm.indices()[internal::random<int>(0, i)]); + Fixed upper = Fixed::Random().template triangularView<Upper>(); + RealScalar factorial(1); + for (int i = 0; i < N; ++i) { + upper(i, i) = Scalar(RealScalar(i + 1)); + factorial *= RealScalar(i + 1); + } + const Fixed a = perm * upper; + PartialPivLU<Fixed> lu(a); + VERIFY_IS_EQUAL(lu.matrixLU(), upper); + VERIFY_IS_EQUAL(lu.determinant(), Scalar(RealScalar(perm.determinant()) * factorial)); + check(a); + } + + // Singular: an exactly zero column j stays exactly zero through elimination, so pivot j is exactly zero and every + // earlier one is not. j = 0 and interior j take the zero branch of the pivot search; j = N - 1 takes the last-row + // check. The factorization still satisfies P A = L U, and the determinant is exactly zero. + for (int j : {0, internal::random<int>(0, N - 1), N - 1}) { + Fixed a = Fixed::Random(); + a.col(j).setZero(); + check(a); + VERIFY_IS_EQUAL(first_zero_pivot(a), Index(j)); + VERIFY_IS_EQUAL(PartialPivLU<Fixed>(a).determinant(), Scalar(0)); + } +} + template <typename Scalar, int Options> void lu_subspace_expressions() { using Mat = Matrix<Scalar, 4, 4, Options>; @@ -570,4 +659,20 @@ CALL_SUBTEST_6(lu_blocking_boundary<std::complex<double> >()); CALL_SUBTEST_4(lu_rowmajor_boundary<double>()); CALL_SUBTEST_5(lu_rowmajor_boundary<std::complex<float> >()); + + CALL_SUBTEST_17((lu_partial_piv_fixed<float, 1, ColMajor>())); + CALL_SUBTEST_17((lu_partial_piv_fixed<float, 5, ColMajor>())); + CALL_SUBTEST_17((lu_partial_piv_fixed<float, 12, RowMajor>())); + CALL_SUBTEST_17((lu_partial_piv_fixed<float, 13, RowMajor>())); + CALL_SUBTEST_17((lu_partial_piv_fixed<double, 5, ColMajor>())); + CALL_SUBTEST_17((lu_partial_piv_fixed<double, 7, ColMajor>())); + CALL_SUBTEST_18((lu_partial_piv_fixed<double, 8, ColMajor>())); + CALL_SUBTEST_18((lu_partial_piv_fixed<double, 12, ColMajor>())); + CALL_SUBTEST_18((lu_partial_piv_fixed<double, 13, ColMajor>())); + CALL_SUBTEST_18((lu_partial_piv_fixed<double, 10, RowMajor>())); + CALL_SUBTEST_18((lu_partial_piv_fixed<double, 11, RowMajor>())); + CALL_SUBTEST_19((lu_partial_piv_fixed<std::complex<double>, 7, ColMajor>())); + CALL_SUBTEST_19((lu_partial_piv_fixed<std::complex<float>, 9, RowMajor>())); + CALL_SUBTEST_19((lu_partial_piv_fixed<std::complex<double>, 12, RowMajor>())); + CALL_SUBTEST_19((lu_partial_piv_fixed<std::complex<float>, 11, ColMajor>())); }
diff --git a/test/mixingtypes.cpp b/test/mixingtypes.cpp index c714467..e6850fd 100644 --- a/test/mixingtypes.cpp +++ b/test/mixingtypes.cpp
@@ -109,6 +109,9 @@ VERIFY_IS_APPROX(Eigen::pow(vf.array(), scf), Eigen::pow(vf.template cast<complex<float> >().array(), scf)); VERIFY_IS_APPROX(vf.array().pow(scf), Eigen::pow(vf.template cast<complex<float> >().array(), scf)); VERIFY_MIX_SCALAR(Eigen::pow(scd, vd.array()), Eigen::pow(scd, vd.template cast<complex<double> >().array())); + // The complex-exponent functor has no packetOp, so this assignment must not vectorize. + Vec_cf powcf = vf.array().pow(scf).matrix(); + VERIFY_IS_APPROX(powcf.array(), Eigen::pow(vf.template cast<complex<float> >().array(), scf)); // check dot product vf.dot(vf);
diff --git a/test/no_automatic_resizing.cpp b/test/no_automatic_resizing.cpp index 66d1a78..13f03f5 100644 --- a/test/no_automatic_resizing.cpp +++ b/test/no_automatic_resizing.cpp
@@ -8,11 +8,12 @@ // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. // SPDX-License-Identifier: MPL-2.0 -// Test that EIGEN_NO_AUTOMATIC_RESIZING still allows assignment to -// default-constructed (empty) matrices and arrays. +// Test resizing restrictions and assignment to empty matrices and arrays. #define EIGEN_NO_AUTOMATIC_RESIZING #include "main.h" +#include <Eigen/Core> +#include <Eigen/SparseCore> template <typename Scalar> void testNoAutomaticResizing() { @@ -100,8 +101,137 @@ } } +// Isolate this check from the additional shape assertions in assignment evaluators. +template <typename PlainObject> +struct ResizeToMatchProbe : PlainObject { + using PlainObject::_resize_to_match; +}; + +template <typename Vector> +void testResizeToMatchVector() { + ResizeToMatchProbe<Vector> dst; + MatrixXd source = MatrixXd::Ones(2, 3); + VERIFY_RAISES_ASSERT(dst._resize_to_match(source)); + dst.resize(6); + dst.setConstant(-1); + VERIFY_RAISES_ASSERT(dst._resize_to_match(source)); + VERIFY_IS_EQUAL(dst.sum(), -6); + + // A non-empty vector accepts a same-length source of either runtime orientation and keeps its shape. + source.resize(6, 1); + dst._resize_to_match(source); + source.resize(1, 6); + dst._resize_to_match(source); + VERIFY_IS_EQUAL(dst.rows(), Vector::RowsAtCompileTime == 1 ? 1 : 6); + VERIFY_IS_EQUAL(dst.cols(), Vector::RowsAtCompileTime == 1 ? 6 : 1); + source.resize(1, 5); + VERIFY_RAISES_ASSERT(dst._resize_to_match(source)); + + ResizeToMatchProbe<Vector> empty; + empty._resize_to_match(source); + VERIFY_IS_EQUAL(empty.size(), 5); + VERIFY_IS_EQUAL(empty.rows(), Vector::RowsAtCompileTime == 1 ? 1 : 5); + VERIFY_IS_EQUAL(empty.cols(), Vector::RowsAtCompileTime == 1 ? 5 : 1); +} + +template <int Order> +void testVectorOrientationNoAutomaticResizing() { + using Matrix = Eigen::Matrix<double, Dynamic, Dynamic, Order>; + using Array = Eigen::Array<double, Dynamic, Dynamic, Order>; + const Matrix column = Matrix::Constant(3, 1, 4.0); + const Matrix row = Matrix::Constant(1, 3, 5.0); + RowVectorXd r = RowVectorXd::Constant(3, -1.0); + VectorXd c = VectorXd::Constant(3, -1.0); + VERIFY_RAISES_ASSERT(r = column); + VERIFY_RAISES_ASSERT(c = row); + VERIFY_IS_EQUAL(r.sum(), -3.0); + VERIFY_IS_EQUAL(c.sum(), -3.0); + + Eigen::Array<double, 1, Dynamic> ar = Eigen::Array<double, 1, Dynamic>::Constant(3, -1.0); + ArrayXd ac = ArrayXd::Constant(3, -1.0); + const Array arrayColumn = Array::Constant(3, 1, 4.0); + const Array arrayRow = Array::Constant(1, 3, 5.0); + VERIFY_RAISES_ASSERT(ar = arrayColumn); + VERIFY_RAISES_ASSERT(ac = arrayRow); + + const Matrix identity = Matrix::Identity(3, 3); + VERIFY_RAISES_ASSERT(r.noalias() = identity * column); + VERIFY_RAISES_ASSERT(c.noalias() = row * identity); + VERIFY_RAISES_ASSERT(r.noalias() = 2.0 * (identity * column)); + VERIFY_RAISES_ASSERT(c.noalias() = 2.0 * (row * identity)); + + // Compile-time vector orientation conversion remains supported. + c << 1, 2, 3; + r = c; + VERIFY_IS_EQUAL(r, RowVector3d(1, 2, 3)); + c = RowVector3d(4, 5, 6); + VERIFY_IS_EQUAL(c, Vector3d(4, 5, 6)); + ac << 1, 2, 3; + ar = ac; + VERIFY_IS_EQUAL(ar.matrix(), RowVector3d(1, 2, 3)); + ac = ar; + VERIFY_IS_EQUAL(ac.matrix(), Vector3d(1, 2, 3)); + r = row; + c = column; + VERIFY_IS_EQUAL(r.sum(), 15.0); + VERIFY_IS_EQUAL(c.sum(), 12.0); +} + +template <int Order> +void testSparseNoAutomaticResizing() { + SparseMatrix<double, Order> row(1, 3), column(3, 1); + row.insert(0, 2) = 7.0; + column.insert(2, 0) = 8.0; + VectorXd c = VectorXd::Constant(3, -1.0); + RowVectorXd r = RowVectorXd::Constant(3, -1.0); + VERIFY_RAISES_ASSERT(c = row); + VERIFY_RAISES_ASSERT(r = column); + c = column; + r = row; + VERIFY_IS_EQUAL(c, Vector3d(0, 0, 8)); + VERIFY_IS_EQUAL(r, RowVector3d(0, 0, 7)); +} + +template <typename = void> +void testProductNoAutomaticResizing() { + const MatrixXd lhs = MatrixXd::Ones(3, 2); + const MatrixXd rhs = MatrixXd::Constant(2, 4, 2.0); + const MatrixXd expected = MatrixXd::Constant(3, 4, 4.0); + + MatrixXd dst; + dst.noalias() = lhs * rhs; + VERIFY_IS_APPROX(dst, expected); + dst.noalias() = 2.0 * (lhs * rhs); + VERIFY_IS_APPROX(dst, 2.0 * expected); + dst.noalias() = lhs * rhs; + VERIFY_IS_APPROX(dst, expected); + + dst.resize(2, 6); // Same coefficient count does not imply compatible dimensions. + dst.setOnes(); + VERIFY_RAISES_ASSERT(dst.noalias() = lhs * rhs); + VERIFY_RAISES_ASSERT(dst.noalias() = 2.0 * (lhs * rhs)); + VERIFY_IS_EQUAL(dst.rows(), 2); + VERIFY_IS_EQUAL(dst.cols(), 6); + VERIFY_IS_EQUAL(dst.sum(), 12.0); + + RowVectorXd row; + row.noalias() = lhs * Vector2d::Ones(); + VERIFY_IS_APPROX(row, RowVector3d::Constant(2.0)); + row.resize(2); + VERIFY_RAISES_ASSERT(row.noalias() = lhs * Vector2d::Ones()); +} + EIGEN_DECLARE_TEST(no_automatic_resizing) { CALL_SUBTEST_1(testNoAutomaticResizing<float>()); CALL_SUBTEST_2(testNoAutomaticResizing<double>()); + CALL_SUBTEST_2(testProductNoAutomaticResizing<>()); + CALL_SUBTEST_2(testVectorOrientationNoAutomaticResizing<ColMajor>()); + CALL_SUBTEST_2(testVectorOrientationNoAutomaticResizing<RowMajor>()); + CALL_SUBTEST_2(testResizeToMatchVector<VectorXd>()); + CALL_SUBTEST_2(testResizeToMatchVector<RowVectorXd>()); + CALL_SUBTEST_2(testResizeToMatchVector<ArrayXd>()); + CALL_SUBTEST_2((testResizeToMatchVector<Eigen::Array<double, 1, Dynamic>>())); CALL_SUBTEST_3(testNoAutomaticResizing<std::complex<double>>()); + CALL_SUBTEST_4(testSparseNoAutomaticResizing<ColMajor>()); + CALL_SUBTEST_4(testSparseNoAutomaticResizing<RowMajor>()); }
diff --git a/test/numerical_test_helpers.cpp b/test/numerical_test_helpers.cpp new file mode 100644 index 0000000..5e1d96b --- /dev/null +++ b/test/numerical_test_helpers.cpp
@@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include "packetmath_test_shared.h" +#include <Eigen/Geometry> +#include <Eigen/SparseCore> + +template <typename Real> +void comparison_helpers() { + const Real inf = NumTraits<Real>::infinity(); + const Real nan = NumTraits<Real>::quiet_NaN(); + VERIFY(test_isCwiseApprox(inf, inf, true)); + VERIFY(!test_isCwiseApprox(inf, -inf, false)); + VERIFY(!test_isApprox(inf, -inf)); + VERIFY(test_isApprox(inf, inf)); + VERIFY(test_isApprox(nan, nan)); + VERIFY(!test_isMuchSmallerThan(inf, inf)); + VERIFY(!test_isMuchSmallerThan(nan, nan)); + VERIFY(!test_isApproxOrLessThan(inf, -inf)); + VERIFY(!test_isApproxOrLessThan(nan, nan)); + VERIFY(test_isApproxOrLessThan(inf, inf)); + VERIFY(test_isCwiseApprox(nan, nan, true)); + VERIFY(!test_isCwiseApprox(nan, Real(0), false)); + VERIFY(!test_isCwiseApprox(inf, NumTraits<Real>::highest(), false)); + VERIFY(test_isCwiseApprox(Real(0), Real(-0.0), true)); + VERIFY(!test_isCwiseApprox(Real(1), Real(2), false)); + const Real eps = NumTraits<Real>::epsilon(); + const Real near = Real(1) + Real(8) * eps; + VERIFY(test_isCwiseApprox(Real(1), near, false, Real(8) * eps)); + VERIFY(!test_isCwiseApprox(Real(1), near, false, Real(7) * eps)); + VERIFY(test_isCwiseApprox(Real(1), Real(1) + test_precision<Real>() / Real(2), false)); + + using Complex = std::complex<Real>; + VERIFY(test_isApprox(Complex(1), Real(1))); + VERIFY(test_isApprox(Real(1), Complex(1))); + VERIFY(test_isMuchSmallerThan(Complex(0), Real(1))); + VERIFY(test_isMuchSmallerThan(Real(0), Complex(1))); + using ArrayType = Array<Complex, 2, 2>; + ArrayType x, y; + x << Complex(inf), Complex(nan), Complex(1), Complex(0); + y = x; + VERIFY(test_isCwiseApprox(x, y, true)); + VERIFY(test_isCwiseApprox(x, y, false)); + y(1, 0) = Complex(1, test_precision<Real>() / Real(2)); + VERIFY(test_isCwiseApprox(x, y, false)); + VERIFY(!test_isCwiseApprox(x, y, true)); + y(0, 0) = Complex(-inf); + VERIFY(!test_isCwiseApprox(x, y, false)); + VERIFY(!test_isCwiseApprox(x, y.row(0), false)); + VERIFY(!test_isApprox(Complex(inf), Complex(inf))); + VERIFY(!test_isApprox(Complex(nan), Complex(nan))); + + for (Real scale : + {Real(1), NumTraits<Real>::highest() / Real(32), (numext::numeric_limits<Real>::min)() * Real(32)}) { + x.setConstant(Complex(scale)); + y = Real(2) * x; + VERIFY(!test_isCwiseApprox(x, y, false)); + VERIFY(!test_isApprox(Complex(scale), Complex(Real(2) * scale))); + VERIFY(!test_isMuchSmallerThan(Complex(scale), Complex(Real(2) * scale))); + VERIFY(test_isMuchSmallerThan(x * Real(0), y)); + VERIFY(test_isMuchSmallerThan(x * Real(0), scale)); + VERIFY_IS_EQUAL(test_relative_error(x, y), Real(1)); + VERIFY_IS_EQUAL(test_relative_error(x.real(), y), Real(1)); + VERIFY_IS_EQUAL(test_relative_error(scale, Real(2) * scale), Real(1)); + } + + x.setZero(); + y.setZero(); + VERIFY_IS_EQUAL(test_relative_error(x, y), Real(0)); + VERIFY_IS_EQUAL(test_relative_error(Real(0), Real(0)), Real(0)); + y.setOnes(); + VERIFY((numext::isinf)(test_relative_error(x, y))); + const SparseMatrix<Complex> sx = x.matrix().sparseView(); + const SparseMatrix<Complex> sy = y.matrix().sparseView(); + VERIFY(test_isCwiseApprox(sx, sx, true)); + VERIFY(!test_isCwiseApprox(sx, sy, false)); + + using Vector = Matrix<Real, 2, 1>; + ParametrizedLine<Real, 2> a(Vector::Ones(), Vector::UnitX()); + ParametrizedLine<Real, 2> b(Vector::Ones(), -Vector::UnitX()); + VERIFY_IS_EQUAL(test_relative_error(a, b), Real(2)); +} + +void integer_diagnostics() { + const int low = NumTraits<int>::lowest(); + const int high = NumTraits<int>::highest(); + VERIFY(test_relative_error(low, high) >= 2.0); + VERIFY(test_relative_error(Matrix2i::Constant(low), Matrix2i::Constant(high)) >= 2.0); + VERIFY_IS_EQUAL(test_relative_error(false, false), 0.0f); + VERIFY((numext::isinf)(test_relative_error(false, true))); + VERIFY(test_isCwiseApprox(false, false, true)); + VERIFY(!test_isCwiseApprox(false, true, false)); + VERIFY_IS_EQUAL(test_relative_error(half(1), 2.0f), half(1)); + VERIFY_IS_EQUAL(test_relative_error(bfloat16(1), 2.0f), bfloat16(1)); +} + +template <typename Real> +void scaled_comparison_helpers() { + using MatrixType = Matrix<Real, 2, 2>; + const MatrixType zero = MatrixType::Zero(); + for (Index i = 0; i < zero.size(); ++i) { + MatrixType value = zero; + value(i) = NumTraits<Real>::quiet_NaN(); + VERIFY(!verifyIsApproxScaled(value, zero, Real(1))); + VERIFY(!verifyIsApproxScaled(value.array(), zero.array(), Real(1))); + } + VERIFY(!verifyIsApproxScaled(zero, zero, NumTraits<Real>::infinity())); + VERIFY(!verifyIsApproxScaled(zero, zero, NumTraits<Real>::quiet_NaN())); + const Matrix<Real, Dynamic, Dynamic> empty(0, 3); + VERIFY(verifyIsApproxScaled(empty, empty, Real(1))); +} + +void packet_comparison_helpers() { + const double inf = NumTraits<double>::infinity(); + const double opposite = -inf; + const double one = 1, near = 1 + 1e-8; + VERIFY(test::areApprox(&inf, &inf, 1)); + VERIFY(!test::areApprox(&inf, &opposite, 1)); + VERIFY(!test::areApprox(&inf, &opposite, 1, NumTraits<double>::epsilon())); + // Packet checks retain dummy_precision(), which is tighter than the legacy test_precision(). + VERIFY(!test::areApprox(&one, &near, 1)); +} + +EIGEN_DECLARE_TEST(numerical_test_helpers) { + comparison_helpers<float>(); + comparison_helpers<double>(); + comparison_helpers<long double>(); + integer_diagnostics(); + scaled_comparison_helpers<float>(); + scaled_comparison_helpers<double>(); + packet_comparison_helpers(); +}
diff --git a/test/numerical_test_helpers.h b/test/numerical_test_helpers.h index 137dca8..66d40ac 100644 --- a/test/numerical_test_helpers.h +++ b/test/numerical_test_helpers.h
@@ -33,6 +33,8 @@ bool test_is_equal(const T& actual, const U& expected, bool expect_equal = true); #endif +// Legacy defaults for broad consistency checks, not accuracy bounds. Numerical tests should use explicit +// bounds derived from epsilon, the operation count and conditioning, or a documented ULP budget. template <typename T> inline typename NumTraits<T>::Real test_precision() { return NumTraits<T>::dummy_precision(); @@ -62,23 +64,23 @@ return test_precision<long double>(); } -#define EIGEN_TEST_SCALAR_TEST_OVERLOAD(TYPE) \ - inline bool test_isApprox(TYPE a, TYPE b) { \ - return numext::equal_strict(a, b) || ((numext::isnan)(a) && (numext::isnan)(b)) || \ - (internal::isApprox(a, b, test_precision<TYPE>())); \ - } \ - inline bool test_isCwiseApprox(TYPE a, TYPE b, bool exact) { \ - return numext::equal_strict(a, b) || ((numext::isnan)(a) && (numext::isnan)(b)) || \ - (!exact && internal::isApprox(a, b, test_precision<TYPE>())); \ - } \ - inline bool test_isMuchSmallerThan(TYPE a, TYPE b) { \ - return internal::isMuchSmallerThan(a, b, test_precision<TYPE>()); \ - } \ - inline bool test_isApproxOrLessThan(TYPE a, TYPE b) { \ - return internal::isApproxOrLessThan(a, b, test_precision<TYPE>()); \ +#define EIGEN_TEST_SCALAR_TEST_OVERLOAD(TYPE) \ + inline bool test_isCwiseApprox(TYPE a, TYPE b, bool exact, \ + typename NumTraits<TYPE>::Real precision = test_precision<TYPE>()) { \ + return numext::equal_strict(a, b) || ((numext::isnan)(a) && (numext::isnan)(b)) || \ + (!exact && (numext::isfinite)(a) && (numext::isfinite)(b) && internal::isApprox(a, b, precision)); \ + } \ + inline bool test_isApprox(TYPE a, TYPE b) { return test_isCwiseApprox(a, b, false); } \ + inline bool test_isMuchSmallerThan(TYPE a, TYPE b) { \ + return (numext::isfinite)(a) && internal::isMuchSmallerThan(a, b, test_precision<TYPE>()); \ + } \ + inline bool test_isApproxOrLessThan(TYPE a, TYPE b) { \ + return a <= b || \ + ((numext::isfinite)(a) && (numext::isfinite)(b) && internal::isApprox(a, b, test_precision<TYPE>())); \ } EIGEN_TEST_SCALAR_TEST_OVERLOAD(short) +EIGEN_TEST_SCALAR_TEST_OVERLOAD(bool) EIGEN_TEST_SCALAR_TEST_OVERLOAD(unsigned short) EIGEN_TEST_SCALAR_TEST_OVERLOAD(int) EIGEN_TEST_SCALAR_TEST_OVERLOAD(unsigned int) @@ -91,67 +93,79 @@ EIGEN_TEST_SCALAR_TEST_OVERLOAD(half) EIGEN_TEST_SCALAR_TEST_OVERLOAD(bfloat16) +#ifndef EIGEN_TEST_NO_LONGDOUBLE +EIGEN_TEST_SCALAR_TEST_OVERLOAD(long double) +#endif + #undef EIGEN_TEST_SCALAR_TEST_OVERLOAD #ifndef EIGEN_TEST_NO_COMPLEX -inline bool test_isApprox(const std::complex<float>& a, const std::complex<float>& b) { - return internal::isApprox(a, b, test_precision<std::complex<float>>()); -} -inline bool test_isMuchSmallerThan(const std::complex<float>& a, const std::complex<float>& b) { - return internal::isMuchSmallerThan(a, b, test_precision<std::complex<float>>()); -} +#define EIGEN_TEST_COMPLEX_TEST_OVERLOAD(REAL) \ + inline bool test_isApprox(const std::complex<REAL>& a, const std::complex<REAL>& b) { \ + using Vector = Matrix<std::complex<REAL>, 1, 1>; \ + return Vector::Constant(a).isApprox(Vector::Constant(b), test_precision<std::complex<REAL>>()); \ + } \ + inline bool test_isMuchSmallerThan(const std::complex<REAL>& a, const std::complex<REAL>& b) { \ + using Vector = Matrix<std::complex<REAL>, 1, 1>; \ + return Vector::Constant(a).isMuchSmallerThan(Vector::Constant(b), test_precision<std::complex<REAL>>()); \ + } \ + inline bool test_isCwiseApprox(const std::complex<REAL>& a, const std::complex<REAL>& b, bool exact, \ + REAL precision = test_precision<std::complex<REAL>>()) { \ + using Vector = Matrix<std::complex<REAL>, 1, 1>; \ + return numext::equal_strict(a, b) || ((numext::isnan)(a) && (numext::isnan)(b)) || \ + (!exact && Vector::Constant(a).isApprox(Vector::Constant(b), precision)); \ + } -inline bool test_isApprox(const std::complex<double>& a, const std::complex<double>& b) { - return internal::isApprox(a, b, test_precision<std::complex<double>>()); -} -inline bool test_isMuchSmallerThan(const std::complex<double>& a, const std::complex<double>& b) { - return internal::isMuchSmallerThan(a, b, test_precision<std::complex<double>>()); -} - +EIGEN_TEST_COMPLEX_TEST_OVERLOAD(float) +EIGEN_TEST_COMPLEX_TEST_OVERLOAD(double) #ifndef EIGEN_TEST_NO_LONGDOUBLE -inline bool test_isApprox(const std::complex<long double>& a, const std::complex<long double>& b) { - return internal::isApprox(a, b, test_precision<std::complex<long double>>()); -} -inline bool test_isMuchSmallerThan(const std::complex<long double>& a, const std::complex<long double>& b) { - return internal::isMuchSmallerThan(a, b, test_precision<std::complex<long double>>()); -} +EIGEN_TEST_COMPLEX_TEST_OVERLOAD(long double) #endif +#undef EIGEN_TEST_COMPLEX_TEST_OVERLOAD #endif -#ifndef EIGEN_TEST_NO_LONGDOUBLE -inline bool test_isApprox(const long double& a, const long double& b) { - bool ret = internal::isApprox(a, b, test_precision<long double>()); - if (!ret) - std::cerr << std::endl << " actual = " << a << std::endl << " expected = " << b << std::endl << std::endl; - return ret; -} +template <typename Scalar, bool Scaled> +struct test_relative_error_impl { + template <typename X, typename Y> + static typename NumTraits<Scalar>::Real run(const X& a, const Y& b) { + return numext::sqrt((a.matrix() - b.matrix()).cwiseAbs2().sum() / + numext::mini(a.cwiseAbs2().sum(), b.cwiseAbs2().sum())); + } +}; -inline bool test_isMuchSmallerThan(const long double& a, const long double& b) { - return internal::isMuchSmallerThan(a, b, test_precision<long double>()); -} -inline bool test_isApproxOrLessThan(const long double& a, const long double& b) { - return internal::isApproxOrLessThan(a, b, test_precision<long double>()); -} -#endif // EIGEN_TEST_NO_LONGDOUBLE - -// Boolean subtraction is rejected at compile time, so Boolean coefficients are differenced as int. template <typename Scalar> -using test_difference_scalar_t = std::conditional_t<std::is_same<Scalar, bool>::value, int, Scalar>; +struct test_relative_error_impl<Scalar, true> { + template <typename X, typename Y> + static typename NumTraits<Scalar>::Real run(const X& a, const Y& b) { + using Real = typename NumTraits<Scalar>::Real; + const auto na = internal::scaled_comparison_norm(a); + const auto nb = internal::scaled_comparison_norm(b); + const auto denominator = na <= nb ? na : nb; + const auto difference = internal::scaled_comparison_distance(a, b); + if (difference.fraction == 0 && denominator.fraction == 0) return Real(0); + return Real(numext::ldexp(difference.fraction / denominator.fraction, difference.exponent - denominator.exponent)); + } +}; + +// Promote integers before subtraction and squaring, avoiding Boolean subtraction and integer overflow. Other +// scalars keep their type: a custom complex scalar can inherit a real NonInteger from NumTraits<Real>. +template <typename Scalar> +using test_difference_scalar_t = + std::conditional_t<NumTraits<Scalar>::IsInteger, typename NumTraits<Scalar>::NonInteger, Scalar>; // test_relative_error returns the relative difference between a and b as a real scalar as used in isApprox. template <typename T1, typename T2> typename NumTraits<typename T1::RealScalar>::NonInteger test_relative_error(const EigenBase<T1>& a, const EigenBase<T2>& b) { - using std::sqrt; - typedef typename NumTraits<typename T1::RealScalar>::NonInteger RealScalar; using DiffScalar1 = test_difference_scalar_t<typename T1::Scalar>; using DiffScalar2 = test_difference_scalar_t<typename T2::Scalar>; typename internal::nested_eval<T1, 2>::type ea(a.derived()); typename internal::nested_eval<T2, 2>::type eb(b.derived()); - return sqrt( - RealScalar( - (ea.matrix().template cast<DiffScalar1>() - eb.matrix().template cast<DiffScalar2>()).cwiseAbs2().sum()) / - RealScalar((std::min)(eb.cwiseAbs2().sum(), ea.cwiseAbs2().sum()))); + // Exponent scaling needs binary floating-point on both sides. + constexpr bool kScaled = + internal::use_scaled_comparison<DiffScalar1>::value && internal::use_scaled_comparison<DiffScalar2>::value; + return test_relative_error_impl<DiffScalar1, kScaled>::run(ea.template cast<DiffScalar1>(), + eb.template cast<DiffScalar2>()); } template <typename T1, typename T2> @@ -171,7 +185,7 @@ template <typename S, int D, int O> S test_relative_error(const ParametrizedLine<S, D, O>& a, const ParametrizedLine<S, D, O>& b) { - return (std::max)(test_relative_error(a.origin(), b.origin()), test_relative_error(a.origin(), b.origin())); + return (std::max)(test_relative_error(a.origin(), b.origin()), test_relative_error(a.direction(), b.direction())); } template <typename S, int D> @@ -203,9 +217,10 @@ template <typename T1, typename T2, std::enable_if_t<internal::is_arithmetic<typename NumTraits<T1>::Real>::value, int> = 0> typename NumTraits<typename NumTraits<T1>::Real>::NonInteger test_relative_error(const T1& a, const T2& b) { - typedef typename NumTraits<typename NumTraits<T1>::Real>::NonInteger RealScalar; - return numext::sqrt(RealScalar(numext::abs2(a - b)) / - (numext::mini)(RealScalar(numext::abs2(a)), RealScalar(numext::abs2(b)))); + using Scalar = decltype(a - b); + using Real = typename NumTraits<typename NumTraits<T1>::Real>::NonInteger; + using Vector = Matrix<Scalar, 1, 1>; + return Real(test_relative_error(Vector::Constant(Scalar(a)), Vector::Constant(Scalar(b)))); } template <typename T> @@ -265,17 +280,14 @@ // Largest coefficient magnitude of a matrix or of an array expression. template <typename Derived> -typename NumTraits<typename Derived::Scalar>::Real max_abs_coeff(const MatrixBase<Derived>& m) { - return m.cwiseAbs().maxCoeff(); -} -template <typename Derived> -typename NumTraits<typename Derived::Scalar>::Real max_abs_coeff(const ArrayBase<Derived>& a) { - return a.abs().maxCoeff(); +typename NumTraits<typename Derived::Scalar>::Real max_abs_coeff(const DenseBase<Derived>& m) { + using Real = typename NumTraits<typename Derived::Scalar>::Real; + return m.size() == 0 ? Real(0) : m.derived().matrix().cwiseAbs().template maxCoeff<PropagateNaN>(); } -// Matrix and array counterpart of test_isApproxWithRef: compares two expressions that are mathematically equal but -// whose evaluations differ by rounding proportional to `scale` rather than to the result. verifyIsApprox measures the -// error relative to the result, which no implementation can meet once the result is formed by cancellation. +// Compares two expressions that are mathematically equal but whose evaluations differ by rounding proportional to +// `scale` rather than to the result. verifyIsApprox measures the error relative to the result, which no +// implementation can meet once the result is formed by cancellation. template <typename Type1, typename Type2> inline bool verifyIsApproxScaled(const Type1& a, const Type2& b, const typename NumTraits<typename Type1::Scalar>::Real& scale) { @@ -295,25 +307,13 @@ return true; } -// The idea behind this function is to compare the two scalars a and b where -// the scalar ref is a hint about the expected order of magnitude of a and b. -// WARNING: the scalar a and b must be positive -// Therefore, if a and b happen to be very small compared to ref, -// we won't issue a false negative. -// This test could be: abs(a-b) <= eps * ref -// However, it seems that simply comparing a+ref and b+ref is more sensitive to true error. -template <typename Scalar, typename ScalarRef> -inline bool test_isApproxWithRef(const Scalar& a, const Scalar& b, const ScalarRef& ref) { - return test_isApprox(a + ref, b + ref); -} - template <typename Derived1, typename Derived2> -inline bool test_isMuchSmallerThan(const MatrixBase<Derived1>& m1, const MatrixBase<Derived2>& m2) { +inline bool test_isMuchSmallerThan(const DenseBase<Derived1>& m1, const DenseBase<Derived2>& m2) { return m1.isMuchSmallerThan(m2, test_precision<typename internal::traits<Derived1>::Scalar>()); } template <typename Derived> -inline bool test_isMuchSmallerThan(const MatrixBase<Derived>& m, +inline bool test_isMuchSmallerThan(const DenseBase<Derived>& m, const typename NumTraits<typename internal::traits<Derived>::Scalar>::Real& s) { return m.isMuchSmallerThan(s, test_precision<typename internal::traits<Derived>::Scalar>()); } @@ -323,7 +323,15 @@ return m.isUnitary(test_precision<typename internal::traits<Derived>::Scalar>()); } -// Checks component-wise, works with infs and nans. +// Checks component-wise, works with infs and nans. A scalar without a dedicated overload above, such as a custom +// complex type over a built-in real, delegates to its own test_isApprox. +template <typename Scalar, + std::enable_if_t< + !std::is_base_of<EigenBase<Scalar>, Scalar>::value && !internal::is_arithmetic<Scalar>::value, int> = 0> +bool test_isCwiseApprox(const Scalar& a, const Scalar& b, bool exact) { + return numext::equal_strict(a, b) || ((numext::isnan)(a) && (numext::isnan)(b)) || (!exact && test_isApprox(a, b)); +} + template <typename Derived1, typename Derived2> bool test_isCwiseApprox(const DenseBase<Derived1>& m1, const DenseBase<Derived2>& m2, bool exact) { if (m1.rows() != m2.rows()) { @@ -334,8 +342,7 @@ } for (Index r = 0; r < m1.rows(); ++r) { for (Index c = 0; c < m1.cols(); ++c) { - if (m1(r, c) != m2(r, c) && !((numext::isnan)(m1(r, c)) && (numext::isnan)(m2(r, c))) && - (exact || !test_isApprox(m1(r, c), m2(r, c)))) { + if (!test_isCwiseApprox(m1(r, c), m2(r, c), exact)) { return false; } }
diff --git a/test/numext.cpp b/test/numext.cpp index 053aa95..c2bac68 100644 --- a/test/numext.cpp +++ b/test/numext.cpp
@@ -438,47 +438,26 @@ VERIFY_IS_APPROX(rsqrtx * rsqrtx, invx); } -// GCC and MSVC differ in their treatment of 1/(0 + 0i) -// GCC/clang = (inf, nan) -// MSVC = (nan, nan) -// and 1 / (x + inf i) -// GCC/clang = (0, 0) -// MSVC = (nan, nan) -#if (EIGEN_COMP_GNUC) - { - const int kNumCorners = 20; - const ComplexT corners[kNumCorners][2] = { - // Only consistent across GCC, clang - {ComplexT(zero, zero), ComplexT(zero, zero)}, - {ComplexT(-zero, zero), ComplexT(zero, zero)}, - {ComplexT(zero, -zero), ComplexT(zero, zero)}, - {ComplexT(-zero, -zero), ComplexT(zero, zero)}, - {ComplexT(one, inf), ComplexT(inf, inf)}, - {ComplexT(nan, inf), ComplexT(inf, inf)}, - {ComplexT(one, -inf), ComplexT(inf, -inf)}, - {ComplexT(nan, -inf), ComplexT(inf, -inf)}, - // Consistent across GCC, clang, MSVC - {ComplexT(-inf, one), ComplexT(zero, inf)}, - {ComplexT(inf, one), ComplexT(inf, zero)}, - {ComplexT(-inf, -one), ComplexT(zero, -inf)}, - {ComplexT(inf, -one), ComplexT(inf, -zero)}, - {ComplexT(-inf, nan), ComplexT(nan, inf)}, - {ComplexT(inf, nan), ComplexT(inf, nan)}, - {ComplexT(zero, nan), ComplexT(nan, nan)}, - {ComplexT(one, nan), ComplexT(nan, nan)}, - {ComplexT(nan, zero), ComplexT(nan, nan)}, - {ComplexT(nan, one), ComplexT(nan, nan)}, - {ComplexT(nan, -one), ComplexT(nan, nan)}, - {ComplexT(nan, nan), ComplexT(nan, nan)}, - }; + // Explicit values: complex division in a 1/sqrt reference is compiler-specific and changes under fast-math. + const int kNumCorners = 20; + const ComplexT corners[kNumCorners][2] = { + {ComplexT(zero, zero), ComplexT(inf, nan)}, {ComplexT(-zero, zero), ComplexT(inf, nan)}, + {ComplexT(zero, -zero), ComplexT(inf, nan)}, {ComplexT(-zero, -zero), ComplexT(inf, nan)}, + {ComplexT(one, inf), ComplexT(zero, -zero)}, {ComplexT(nan, inf), ComplexT(zero, -zero)}, + {ComplexT(one, -inf), ComplexT(zero, zero)}, {ComplexT(nan, -inf), ComplexT(zero, zero)}, + {ComplexT(-inf, one), ComplexT(zero, -zero)}, {ComplexT(inf, one), ComplexT(zero, -zero)}, + {ComplexT(-inf, -one), ComplexT(zero, zero)}, {ComplexT(inf, -one), ComplexT(zero, zero)}, + {ComplexT(-inf, nan), ComplexT(zero, -zero)}, {ComplexT(inf, nan), ComplexT(zero, -zero)}, + {ComplexT(zero, nan), ComplexT(nan, nan)}, {ComplexT(one, nan), ComplexT(nan, nan)}, + {ComplexT(nan, zero), ComplexT(nan, nan)}, {ComplexT(nan, one), ComplexT(nan, nan)}, + {ComplexT(nan, -one), ComplexT(nan, nan)}, {ComplexT(nan, nan), ComplexT(nan, nan)}, + }; - for (int i = 0; i < kNumCorners; ++i) { - const ComplexT& x = corners[i][0]; - const ComplexT rsqrtx = ComplexT(one, zero) / corners[i][1]; - VERIFY_IS_EQUAL_OR_NANS(numext::rsqrt(x), rsqrtx); - } + for (int i = 0; i < kNumCorners; ++i) { + const ComplexT& x = corners[i][0]; + const ComplexT& rsqrtx = corners[i][1]; + VERIFY_IS_EQUAL_OR_NANS(numext::rsqrt(x), rsqrtx); } -#endif } }; @@ -669,6 +648,50 @@ } } +template <typename Scalar> +EIGEN_DONT_INLINE void check_binary_sign_no_flush_values() { + using Binary = internal::binary_floating_point_traits<Scalar>; + using Bits = typename Binary::Bits; + const Bits sign = Binary::kSignBit; + const Bits min_normal = Binary::kExponentUnit; + const Bits infinity = Binary::kExponentMask; + const Bits nan = infinity | (min_normal >> 1) | Bits(0x12345); + const Bits one = Binary::bits(Scalar(1)); + const Bits input_bits[] = {Bits(0), + sign, + Bits(1), + sign | Bits(1), + min_normal - 1, + sign | (min_normal - 1), + min_normal, + sign | min_normal, + one, + sign | one, + infinity, + sign | infinity, + nan, + sign | nan}; + const Bits expected_bits[] = {Bits(0), Bits(0), one, sign | one, one, sign | one, one, + sign | one, one, sign | one, one, sign | one, nan, sign | nan}; + for (int i = 0; i < int(sizeof(input_bits) / sizeof(input_bits[0])); ++i) { + const Scalar input = numext::bit_cast<Scalar>(input_bits[i]); + VERIFY_IS_EQUAL(numext::bit_cast<Bits>(numext::sign(input)), expected_bits[i]); + } +} + +template <typename Scalar> +void check_binary_sign_no_flush() { + check_binary_sign_no_flush_values<Scalar>(); + ScopedFlushToZero flush; + if (!flush.isSupported()) return; + check_binary_sign_no_flush_values<Scalar>(); +#if !defined(EIGEN_GPU_COMPILE_PHASE) && !defined(SYCL_DEVICE_ONLY) && EIGEN_ARCH_i386_OR_x86_64 && \ + defined(_MM_SET_DENORMALS_ZERO_MODE) + _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON); + check_binary_sign_no_flush_values<Scalar>(); +#endif +} + template <typename T> void check_ceil_power_of_two() { const T zero(0); @@ -832,6 +855,8 @@ CALL_SUBTEST(check_zero_no_flush<float>()); CALL_SUBTEST(check_zero_no_flush<double>()); CALL_SUBTEST(check_zero_no_flush_custom_scalar()); + CALL_SUBTEST(check_binary_sign_no_flush<float>()); + CALL_SUBTEST(check_binary_sign_no_flush<double>()); CALL_SUBTEST(check_shift<int8_t>()); CALL_SUBTEST(check_shift<int16_t>());
diff --git a/test/packet_segment.cpp b/test/packet_segment.cpp index a18ce37..3c16766 100644 --- a/test/packet_segment.cpp +++ b/test/packet_segment.cpp
@@ -119,9 +119,42 @@ verify_data(aligned_data_in, aligned_data_out, b, begin, count); } + static void test_all_ranges() { + // every (begin, count) with begin + count <= PacketSize: lanes in the range are copied, the rest are untouched + VectorX<Scalar> data_in(PacketSize), data_out(PacketSize), data_ref(PacketSize); + for (Index begin = 0; begin <= PacketSize; begin++) { + for (Index count = 0; begin + count <= PacketSize; count++) { + data_in.setRandom(); + data_out.setRandom(); + data_ref = data_out; + // A scalar loop: a segment assignment would run the primitives under test. + for (Index i = begin; i < begin + count; ++i) data_ref(i) = data_in(i); + Packet a = internal::ploaduSegment<Packet>(data_in.data(), begin, count); + internal::pstoreuSegment<Scalar, Packet>(data_out.data(), a, begin, count); + VERIFY_IS_CWISE_EQUAL(data_out, data_ref); + } + } + } + static void test_zero_fill() { + // ploaduSegment zeroes the lanes outside [begin, begin + count), even when the memory there holds nonzero values; + // the GEMV kernels sum whole segment packets. + VectorX<Scalar> data_in = VectorX<Scalar>::Constant(PacketSize, Scalar(1)); + VectorX<Scalar> lanes(PacketSize); + for (Index begin = 0; begin <= PacketSize; begin++) { + for (Index count = 0; begin + count <= PacketSize; count++) { + internal::pstoreu(lanes.data(), internal::ploaduSegment<Packet>(data_in.data(), begin, count)); + for (Index i = 0; i < PacketSize; ++i) { + const bool in_range = i >= begin && i < begin + count; + VERIFY_IS_EQUAL(lanes(i), in_range ? Scalar(1) : Scalar(0)); + } + } + } + } static void run() { test_unaligned(); test_aligned(); + test_all_ranges(); + test_zero_fill(); } };
diff --git a/test/packetmath.cpp b/test/packetmath.cpp index 5e72aaa..5acfc0f 100644 --- a/test/packetmath.cpp +++ b/test/packetmath.cpp
@@ -593,6 +593,56 @@ } }; +template <typename Scalar, typename Packet, typename EnableIf = void> +struct packetmath_float_predicates_test { + static void run() {} +}; + +// Floating-point pisnan/pisinf/pisfinite against the scalar predicates over signed zeros, subnormals, infinities and +// NaNs with a payload. Masks are compared bitwise: true is ptrue of the tested type, false is +0. +template <typename Scalar, typename Packet> +struct packetmath_float_predicates_test< + Scalar, Packet, std::enable_if_t<!NumTraits<Scalar>::IsInteger && !NumTraits<Scalar>::IsComplex>> { + static void run() { + using Bits = typename numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type; + const int PacketSize = internal::unpacket_traits<Packet>::size; + Scalar inf = NumTraits<Scalar>::infinity(); + Scalar nan = NumTraits<Scalar>::quiet_NaN(); + Scalar values[] = {Scalar(0), + -Scalar(0), + Scalar(1), + -Scalar(1), + std::numeric_limits<Scalar>::denorm_min(), + -std::numeric_limits<Scalar>::denorm_min(), + (std::numeric_limits<Scalar>::min)(), + NumTraits<Scalar>::highest(), + NumTraits<Scalar>::lowest(), + inf, + -inf, + nan, + -nan, + numext::bit_cast<Scalar>(Bits(numext::bit_cast<Bits>(nan) | Bits(1)))}; + const int num_values = sizeof(values) / sizeof(values[0]); + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar data[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar res[PacketSize]; + EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar lane_true[PacketSize]; + internal::pstore(lane_true, internal::ptrue(internal::pset1<Packet>(Scalar(0)))); + auto mask = [&](int i, bool expected) { return expected ? numext::bit_cast<Bits>(lane_true[i]) : Bits(0); }; + for (int start = 0; start < num_values; start += PacketSize) { + for (int i = 0; i < PacketSize; ++i) data[i] = values[(start + i) % num_values]; + internal::pstore(res, internal::pisnan(internal::pload<Packet>(data))); + for (int i = 0; i < PacketSize; ++i) + VERIFY(numext::bit_cast<Bits>(res[i]) == mask(i, (numext::isnan)(data[i])) && "pisnan"); + internal::pstore(res, internal::pisinf(internal::pload<Packet>(data))); + for (int i = 0; i < PacketSize; ++i) + VERIFY(numext::bit_cast<Bits>(res[i]) == mask(i, (numext::isinf)(data[i])) && "pisinf"); + internal::pstore(res, internal::pisfinite(internal::pload<Packet>(data))); + for (int i = 0; i < PacketSize; ++i) + VERIFY(numext::bit_cast<Bits>(res[i]) == mask(i, (numext::isfinite)(data[i])) && "pisfinite"); + } + } +}; + template <typename Scalar, typename Packet, typename = void> struct packetmath_64bit_boundary_test { static void run() {} @@ -623,7 +673,7 @@ CHECK_CWISE2_IF(internal::packet_traits<Scalar>::HasMax, (std::max), internal::pmax); CHECK_CWISE2_IF(internal::packet_traits<Scalar>::HasMul, test::REF_MUL, internal::pmul); CHECK_CWISE1_IF(internal::packet_traits<Scalar>::HasNegate, test::negate, internal::pnegate); - CHECK_CWISE1(ref_abs, internal::pabs); + CHECK_CWISE1_IF(internal::packet_traits<Scalar>::HasAbs, ref_abs, internal::pabs); }; constexpr Scalar high = 0x11111111; @@ -961,9 +1011,10 @@ packetmath_pcast_ops_runner<Scalar, Packet>::run(); packetmath_minus_zero_add_test<Scalar, Packet>::run(); packetmath_integer_predicates_test<Scalar, Packet>::run(); + packetmath_float_predicates_test<Scalar, Packet>::run(); packetmath_64bit_boundary_test<Scalar, Packet>::run(); - CHECK_CWISE3_IF(true, REF_MADD, internal::pmadd); + CHECK_CWISE3_IF(PacketTraits::HasMul && PacketTraits::HasAdd, REF_MADD, internal::pmadd); if (!std::is_same<Scalar, bool>::value && NumTraits<Scalar>::IsSigned) { nmsub_test<Scalar, Packet>(data1, data2, ref, PacketSize); } @@ -977,8 +1028,8 @@ data1[i + 2 * PacketSize] = Scalar(0) - abs_helper(internal::random<Scalar>()); } if (!std::is_same<Scalar, bool>::value && NumTraits<Scalar>::IsSigned) { - CHECK_CWISE3_IF(true, REF_MSUB, internal::pmsub); - CHECK_CWISE3_IF(true, REF_NMADD, internal::pnmadd); + CHECK_CWISE3_IF(PacketTraits::HasMul && PacketTraits::HasSub, REF_MSUB, internal::pmsub); + CHECK_CWISE3_IF(PacketTraits::HasMul && PacketTraits::HasSub, REF_NMADD, internal::pnmadd); } CHECK_CWISE1_IF(PacketTraits::HasSqrt, numext::sqrt, internal::psqrt); @@ -1167,12 +1218,10 @@ data1[0] = Scalar(std::ldexp(Scalar(1.0), NumTraits<Scalar>::max_exponent() - 1)); data1[PacketSize] = Scalar(+NumTraits<Scalar>::min_exponent() - NumTraits<Scalar>::max_exponent()); CHECK_CWISE2_IF(PacketTraits::HasExp, REF_LDEXP, internal::pldexp); - // Near-max magnitude with small negative exponents. Regression guard for - // the 4-way scale-factor split: the remainder factor c2 = 2^(e-3*floor(e/4)) - // is > 1 for e in {-1, -2, -5, -6, ...}, so the multiply tree must apply - // the downscale c1 before c2 -- otherwise (numext::abs(a)) * c2 spuriously - // overflows to inf for finite results like ldexp((numext::numeric_limits) - // <Scalar>::max(), -1). + // Near-max magnitude with small negative exponents: a scale factor above + // one, as the four-factor split's remainder 2^(e - 3 floor(e/4)) was for + // e in {-1, -2, -5, -6, ...}, overflows ldexp(max, -1) to inf when applied + // first. for (int i = 0; i < PacketSize; ++i) { data1[i] = (numext::numeric_limits<Scalar>::max)(); data1[i + PacketSize] = Scalar(-1 - (i % 8)); // -1, -2, ..., -8 @@ -1196,6 +1245,61 @@ data1[i + PacketSize] = Scalar(-2 * NumTraits<Scalar>::max_exponent() - (i % 4)); } CHECK_CWISE2_IF(PacketTraits::HasExp, REF_LDEXP, internal::pldexp); + // An infinite exponent saturates like any exponent past the clamp. ldexp turns an int exponent beyond the range of + // half into +-inf, and vscalef alone gives NaN for 0 * 2^inf and inf * 2^-inf, and a number for NaN * 2^(+-inf). + // The scalar pldexp converts the exponent to int, undefined for inf, so only packets are checked. + if (!internal::is_scalar<Packet>::value && PacketTraits::HasExp) { + const Scalar inf = NumTraits<Scalar>::infinity(), big = (numext::numeric_limits<Scalar>::max)(); + const Scalar bases[] = {Scalar(0), -Scalar(0), Scalar(1), -big, inf, -inf, NumTraits<Scalar>::quiet_NaN()}; + test::packet_helper<PacketTraits::HasExp, Packet> h; + for (const int n : {(std::numeric_limits<int>::max)(), (std::numeric_limits<int>::min)()}) { + for (int k = 0; k < 7; k += PacketSize) { + for (int i = 0; i < PacketSize; ++i) { + data1[i] = bases[(k + i) % 7]; + data1[i + PacketSize] = n > 0 ? inf : -inf; + ref[i] = static_cast<Scalar>(std::ldexp(static_cast<double>(data1[i]), n)); + } + h.store(data2, internal::pldexp(h.load(data1), h.load(data1 + PacketSize))); + for (int i = 0; i < PacketSize; ++i) { + VERIFY((numext::isnan)(ref[i]) ? (numext::isnan)(data2[i]) : test::biteq(data2[i], ref[i])); + } + } + } + } +#if !EIGEN_ARCH_ARM + // Every integer exponent to past both ends of the range, on bases in every sixteenth binade from the smallest + // subnormal up, bit for bit against std::ldexp. Scaling in steps must not round twice: with the last mantissa + // bit dropped by a subnormal intermediate, denorm_min * (1 + eps) / 2 became zero and denorm_min * (1.5 - eps) + // became 2 * denorm_min. + { + const int max_exp = NumTraits<Scalar>::max_exponent(), min_exp = NumTraits<Scalar>::min_exponent(), + digits = NumTraits<Scalar>::digits(); + const Scalar eps = NumTraits<Scalar>::epsilon(); + const Scalar mantissas[] = {Scalar(1), Scalar(1) + eps, Scalar(1.5) - eps, Scalar(2) - eps}; + std::vector<Scalar> bases = {Scalar(0), std::numeric_limits<Scalar>::denorm_min(), + (std::numeric_limits<Scalar>::max)()}; + for (int be = min_exp - digits; be < max_exp; be += 16) { + for (Scalar m : mantissas) bases.push_back(Scalar(std::ldexp(m, be))); + } + test::packet_helper<PacketTraits::HasExp, Packet> h; + const int range = max_exp - min_exp + digits + 20; + for (int e = -range; e <= range; ++e) { + for (size_t k = 0; k < bases.size(); k += PacketSize) { + for (int i = 0; i < PacketSize; ++i) { + const Scalar base = bases[(k + i) % bases.size()]; + data1[i] = (i % 2) ? -base : base; + data1[i + PacketSize] = Scalar(e); // rounded for bfloat16 beyond 256, so read it back + ref[i] = Scalar(std::ldexp(data1[i], static_cast<int>(data1[i + PacketSize]))); + } + h.store(data2, internal::pldexp(h.load(data1), h.load(data1 + PacketSize))); + for (int i = 0; i < PacketSize; ++i) { + VERIFY_IS_EQUAL(data2[i], ref[i]); // prints the values; biteq also tells -0 from +0 + VERIFY(test::biteq(data2[i], ref[i])); + } + } + } + } +#endif } for (int i = 0; i < size; ++i) { @@ -1292,6 +1396,18 @@ VERIFY_IS_APPROX(std::log((std::numeric_limits<Scalar>::min)()), data2[0]); } VERIFY((numext::isnan)(data2[1])); + +#if !EIGEN_ARCH_ARM // 32-bit ARM flushes subnormals. + // Whether to rescale a subnormal input is decided per lane, even beside a positive normal lane. + // bfloat16 is evaluated in float and misses the tolerance by rounding alone, as in the TODOs above. + if (std::numeric_limits<Scalar>::has_denorm == std::denorm_present && !std::is_same<Scalar, bfloat16>::value) { + data1[0] = std::numeric_limits<Scalar>::denorm_min(); + data1[1] = Scalar(2); + h.store(data2, internal::plog(h.load(data1))); + VERIFY_IS_APPROX(std::log(data1[0]), data2[0]); + VERIFY_IS_APPROX(std::log(data1[1]), data2[1]); + } +#endif } if (PacketTraits::HasLog10) { test::packet_helper<PacketTraits::HasLog10, Packet> h; @@ -2240,6 +2356,100 @@ } } +template <typename Scalar> +void packetmath_binary_sign_subnormals() { + using Packet = typename internal::packet_traits<Scalar>::type; + using Binary = internal::binary_floating_point_traits<Scalar>; + using Bits = typename Binary::Bits; + constexpr Index packet_size = internal::unpacket_traits<Packet>::size; + constexpr Index count = 12 * packet_size + 1; + const Bits sign = Binary::kSignBit; + const Bits min_normal = Binary::kExponentUnit; + const Bits infinity = Binary::kExponentMask; + const Bits one = Binary::bits(Scalar(1)); + const Bits nan = infinity | (min_normal >> 1) | Bits(0x12345); + const Bits samples[] = {Bits(0), + sign, + Bits(1), + sign | Bits(1), + min_normal - 1, + sign | (min_normal - 1), + one, + sign | one, + infinity, + sign | infinity, + nan, + sign | nan}; + Array<Scalar, Dynamic, 1> input(count), result(count); + for (Index i = 0; i < count; ++i) input(i) = numext::bit_cast<Scalar>(samples[i % 12]); + input(count - 1) = numext::bit_cast<Scalar>(sign | Bits(1)); // Exercise the scalar tail. + result = input.sign(); + for (Index i = 0; i < count; ++i) { + const Bits bits = numext::bit_cast<Bits>(input(i)); + const Bits magnitude = bits & ~sign; + const Bits expected = magnitude > infinity ? bits : magnitude == 0 ? Bits(0) : (bits & sign) | one; + VERIFY_IS_EQUAL(numext::bit_cast<Bits>(result(i)), expected); + } + + EIGEN_ALIGN_MAX Scalar lanes[packet_size]; + for (Index i = 0; i < packet_size; ++i) lanes[i] = numext::bit_cast<Scalar>(samples[2 + (i % 4)]); + const Packet packet = internal::psign(internal::ploadu<Packet>(lanes)); + internal::pstoreu(lanes, packet); + for (Index i = 0; i < packet_size; ++i) { + const Bits bits = samples[2 + (i % 4)]; + const Bits expected = bits & sign ? sign | one : one; + VERIFY_IS_EQUAL(numext::bit_cast<Bits>(lanes[i]), expected); + } +} + +template <typename Scalar> +void packetmath_binary_sign_flushed() { + ScopedFlushToZero flush; + if (!flush.isSupported()) return; +#if EIGEN_ARCH_i386_OR_x86_64 && defined(_MM_SET_DENORMALS_ZERO_MODE) + _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON); +#endif + VERIFY(ScopedFlushToZero::hardwareFlushesSubnormalInputs()); + packetmath_binary_sign_subnormals<Scalar>(); +} + +#if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512) || defined(EIGEN_VECTORIZE_NEON) || \ + defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) +void packetmath_bfloat16_sign_bits() { + using Packet = internal::packet_traits<bfloat16>::type; + constexpr int packet_size = internal::unpacket_traits<Packet>::size; + EIGEN_ALIGN_MAX bfloat16 input[packet_size], output[packet_size]; + for (unsigned first = 0; first < 65536; first += packet_size) { + for (int i = 0; i < packet_size; ++i) { + input[i] = numext::bit_cast<bfloat16>(static_cast<numext::uint16_t>(first + i)); + } + internal::pstoreu(output, internal::psign(internal::ploadu<Packet>(input))); + for (int i = 0; i < packet_size; ++i) { + const numext::uint16_t bits = static_cast<numext::uint16_t>(first + i); + const numext::uint16_t magnitude = bits & 0x7fff; + numext::uint16_t expected = 0; + if (magnitude > 0x7f80) { + expected = bits; + } else if (magnitude != 0) { + expected = static_cast<numext::uint16_t>((bits & 0x8000) | 0x3f80); + } + VERIFY_IS_EQUAL(numext::bit_cast<numext::uint16_t>(output[i]), expected); + } + } +} + +void packetmath_bfloat16_sign_bits_flushed() { + ScopedFlushToZero flush_to_zero; + if (!flush_to_zero.isSupported()) return; +#if EIGEN_ARCH_i386_OR_x86_64 && defined(_MM_SET_DENORMALS_ZERO_MODE) + // FTZ flushes results; DAZ makes float comparisons read subnormal inputs as zero. + _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON); +#endif + VERIFY(ScopedFlushToZero::hardwareFlushesSubnormalInputs()); + packetmath_bfloat16_sign_bits(); +} +#endif + namespace Eigen { namespace test { @@ -2308,6 +2518,17 @@ packetmath_bfloat16_abs_array(); }); + CALL_SUBTEST_1(packetmath_binary_sign_subnormals<float>()); + CALL_SUBTEST_2(packetmath_binary_sign_subnormals<double>()); + CALL_SUBTEST_1(packetmath_binary_sign_flushed<float>()); + CALL_SUBTEST_2(packetmath_binary_sign_flushed<double>()); + +#if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512) || defined(EIGEN_VECTORIZE_NEON) || \ + defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) + CALL_SUBTEST_15(packetmath_bfloat16_sign_bits()); + CALL_SUBTEST_15(packetmath_bfloat16_sign_bits_flushed()); +#endif + #if defined(EIGEN_VECTORIZE_RVV10) CALL_SUBTEST_1((packetmath_redux_infinities<float, internal::Packet1Xf>())); CALL_SUBTEST_1((packetmath_redux_infinities<float, internal::Packet2Xf>()));
diff --git a/test/packetmath_fastmath.cpp b/test/packetmath_fastmath.cpp index 8a085e0..bd3b055 100644 --- a/test/packetmath_fastmath.cpp +++ b/test/packetmath_fastmath.cpp
@@ -167,6 +167,50 @@ return Eigen::numext::bit_cast<Eigen::numext::uint32_t>(narrowed); } +template <typename Scalar> +EIGEN_DONT_INLINE void sign_array(const Scalar* input, Scalar* output, Index size) { + Map<Array<Scalar, Dynamic, 1>>(output, size) = Map<const Array<Scalar, Dynamic, 1>>(input, size).sign(); +} + +template <typename Scalar> +void verify_binary_sign_fastmath() { + using Binary = Eigen::internal::binary_floating_point_traits<Scalar>; + using Bits = typename Binary::Bits; + constexpr int count = 2 * Eigen::internal::packet_traits<Scalar>::size + 1; + const Bits sign = Binary::kSignBit; + const Bits min_normal = Binary::kExponentUnit; + const Bits infinity = Binary::kExponentMask; + const Bits one = Binary::bits(Scalar(1)); + const Bits nan = infinity | (min_normal >> 1) | Bits(0x12345); + const Bits samples[] = {Bits(0), + sign, + Bits(1), + sign | Bits(1), + min_normal - 1, + sign | (min_normal - 1), + one, + sign | one, + infinity, + sign | infinity, + nan, + sign | nan}; + Scalar input[count], output[count]; + for (int offset = 0; offset < 12; ++offset) { + for (int lane = 0; lane < count; ++lane) { + const Bits bits = samples[(offset + lane) % 12]; + std::memcpy(static_cast<void*>(input + lane), &bits, sizeof(bits)); + } + // Runtime inputs keep fast-math constant folding out of the operation under test. + sign_array(input, output, count); + for (int lane = 0; lane < count; ++lane) { + const Bits bits = samples[(offset + lane) % 12]; + const Bits magnitude = bits & ~sign; + const Bits expected = magnitude > infinity ? bits : magnitude == 0 ? Bits(0) : (bits & sign) | one; + VERIFY_IS_EQUAL(Eigen::numext::bit_cast<Bits>(output[lane]), expected); + } + } +} + template <typename Scalar, bool Vectorizable = Eigen::internal::packet_traits<Scalar>::Vectorizable> struct packetmath_fastmath_runner { static void run() {} @@ -242,6 +286,33 @@ } }; +template <typename Scalar, typename Packet> +EIGEN_DONT_INLINE void ldexp_packet(const Scalar* a, const Scalar* exponent, Scalar* output) { + Eigen::internal::pstoreu<Scalar, Packet>( + output, Eigen::internal::pldexp(Eigen::internal::ploadu<Packet>(a), Eigen::internal::ploadu<Packet>(exponent))); +} + +// pldexp scales by 2^e in three factors, 2^(t/2) twice and 2^(e - t) once. A product of two factors overflows or +// underflows for |e| beyond the exponent range although a * 2^e is normal, so they must not be reassociated. +template <typename Scalar> +void verify_ldexp_beyond_exponent_range() { + typedef typename Eigen::internal::packet_traits<Scalar>::type Packet; + constexpr int packet_size = Eigen::internal::packet_traits<Scalar>::size; + constexpr int max_exponent = std::numeric_limits<Scalar>::max_exponent; + Scalar a[packet_size], exponent[packet_size], output[packet_size]; + for (int sign : {1, -1}) { + // a * 2^e = 1.5 * 2^(+-(max_exponent - 4)), normal, with |e| = max_exponent + 50. + for (int i = 0; i < packet_size; ++i) { + a[i] = std::ldexp(Scalar(i % 2 ? -1.5 : 1.5), -sign * 54); + exponent[i] = Scalar(sign * (max_exponent + 50)); + } + ldexp_packet<Scalar, Packet>(a, exponent, output); + for (int i = 0; i < packet_size; ++i) { + VERIFY_IS_EQUAL(output[i], std::ldexp(a[i], sign * (max_exponent + 50))); + } + } +} + template <typename Scalar, bool HasIntegerBits = !std::is_void<typename Eigen::numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type>::value> @@ -268,11 +339,15 @@ }; EIGEN_DECLARE_TEST(packetmath_fastmath) { + CALL_SUBTEST(verify_binary_sign_fastmath<float>()); + CALL_SUBTEST(verify_binary_sign_fastmath<double>()); CALL_SUBTEST(packetmath_fastmath_runner<float>::run()); CALL_SUBTEST(packetmath_fastmath_runner<double>::run()); CALL_SUBTEST(packetmath_fastmath_runner<Eigen::half>::run()); CALL_SUBTEST(packetmath_fastmath_runner<Eigen::bfloat16>::run()); CALL_SUBTEST(extended_scalar_constant_runner<long double>::run()); + CALL_SUBTEST(verify_ldexp_beyond_exponent_range<float>()); + CALL_SUBTEST(verify_ldexp_beyond_exponent_range<double>()); #if defined(EIGEN_VECTORIZE_AVX512) #if !defined(EIGEN_VECTORIZE_AVX512FP16)
diff --git a/test/packetmath_test_shared.h b/test/packetmath_test_shared.h index 5f79618..cc4075c 100644 --- a/test/packetmath_test_shared.h +++ b/test/packetmath_test_shared.h
@@ -211,10 +211,10 @@ } template <typename Scalar> -bool areApprox(const Scalar* a, const Scalar* b, int size) { +bool areApprox(const Scalar* a, const Scalar* b, int size, + const typename NumTraits<Scalar>::Real& precision = NumTraits<Scalar>::dummy_precision()) { for (int i = 0; i < size; ++i) { - if (numext::not_equal_strict(a[i], b[i]) && !internal::isApprox(a[i], b[i]) && - !((numext::isnan)(a[i]) && (numext::isnan)(b[i]))) { + if (!test_isCwiseApprox(a[i], b[i], false, precision)) { print_mismatch(a, b, size, i); std::cout << std::setprecision(16) << "Values differ in position " << i << ": " << a[i] << " vs " << b[i] << std::endl; @@ -315,20 +315,6 @@ return true; } -template <typename Scalar> -bool areApprox(const Scalar* a, const Scalar* b, int size, const typename NumTraits<Scalar>::Real& precision) { - for (int i = 0; i < size; ++i) { - if (numext::not_equal_strict(a[i], b[i]) && !internal::isApprox(a[i], b[i], precision) && - !((numext::isnan)(a[i]) && (numext::isnan)(b[i]))) { - print_mismatch(a, b, size, i); - std::cout << std::setprecision(16) << "Values differ in position " << i << ": " << a[i] << " vs " << b[i] - << std::endl; - return false; - } - } - return true; -} - #define CHECK_CWISE1(REFOP, POP) \ { \ for (int i = 0; i < PacketSize; ++i) ref[i] = REFOP(data1[i]); \
diff --git a/test/permutationmatrices.cpp b/test/permutationmatrices.cpp index 00ccc4a..743daeb 100644 --- a/test/permutationmatrices.cpp +++ b/test/permutationmatrices.cpp
@@ -131,6 +131,18 @@ VERIFY_IS_APPROX(A, B.transpose()); } + // A permutation matrix is real and orthogonal: P^H = P^T = P^-1. + { + using LeftPermutationDense = typename LeftPermutationType::DenseMatrixType; + VERIFY_IS_EQUAL(LeftPermutationDense(lp.adjoint().toDenseMatrix()), + LeftPermutationDense(lp.toDenseMatrix().adjoint())); + VERIFY_IS_APPROX(lp.adjoint() * m_permuted, lp.inverse() * m_permuted); + VERIFY_IS_APPROX(m_permuted * rp.adjoint(), m_permuted * rp.inverse()); + VERIFY((lp * lp.adjoint()).toDenseMatrix().isIdentity()); + VERIFY_IS_APPROX(lv.asPermutation().adjoint() * m_permuted, lv.asPermutation().inverse() * m_permuted); + VERIFY_IS_APPROX(MapLeftPerm(lv.data(), lv.size()).adjoint() * m_permuted, lp.inverse() * m_permuted); + } + m_permuted = m_original; lp = lt; rp = rt; @@ -148,6 +160,13 @@ VERIFY_IS_APPROX(m_permuted = m_permuted * rt, m_original * rt); m_permuted = m_original; VERIFY_IS_APPROX(m_permuted = m_permuted * rt.inverse(), m_original * rt.inverse()); + + // Transpositions represent a real permutation: T^H = T^T = T^-1. With lp = lt and rp = rt as above, + // lt * M = lp * M and M * rt = M * rp^T. + VERIFY_IS_APPROX(lt.adjoint() * m_original, lp.inverse() * m_original); + VERIFY_IS_APPROX(m_original * rt.adjoint(), m_original * rp); + m_permuted = m_original; + VERIFY_IS_APPROX(m_permuted = lt.adjoint() * m_permuted, lp.inverse() * m_original); } template <typename T> @@ -253,6 +272,149 @@ } } +// Sums with a dense or diagonal matrix are lazy: the permutation is read as a 0/1 matrix of the other +// operand's scalar type, so no temporary is formed and the results are bit-exact. +template <typename Scalar, int Size> +void permutation_dense_sums(Index size) { + using PermutationType = PermutationMatrix<Size>; + using MatrixType = Matrix<Scalar, Size, Size>; + using VectorType = Matrix<Scalar, Size, 1>; + + Matrix<int, Size, 1> indices; + randomPermutationVector(indices, size); + const PermutationType perm(indices); + const MatrixType permDense = perm.toDenseMatrix().template cast<Scalar>(); + const MatrixType a = MatrixType::Random(size, size); + const DiagonalMatrix<Scalar, Size> diag(VectorType::Random(size)); + const MatrixType diagDense = diag.toDenseMatrix(); + + MatrixType result(size, size); + VERIFY_EVALUATION_COUNT(result.noalias() = a + perm, 0); + VERIFY_IS_EQUAL(result, MatrixType(a + permDense)); + VERIFY_EVALUATION_COUNT(result.noalias() = perm - a, 0); + VERIFY_IS_EQUAL(result, MatrixType(permDense - a)); + result = perm + a; + VERIFY_IS_EQUAL(result, MatrixType(permDense + a)); + result = a - indices.asPermutation(); + VERIFY_IS_EQUAL(result, MatrixType(a - permDense)); + result = diag + perm; + VERIFY_IS_EQUAL(result, MatrixType(diagDense + permDense)); + result = perm - diag; + VERIFY_IS_EQUAL(result, MatrixType(permDense - diagDense)); + result = perm + diag.diagonal().asDiagonal(); + VERIFY_IS_EQUAL(result, MatrixType(permDense + diagDense)); + result = diag - Map<PermutationType>(indices.data(), size); + VERIFY_IS_EQUAL(result, MatrixType(diagDense - permDense)); + result = a + perm + diag; + VERIFY_IS_EQUAL(result, MatrixType(a + permDense + diagDense)); + VERIFY_IS_APPROX((a + perm) * a, (a + permDense) * a); + + // Direct assignment writes each column around its one: compound assignment, the negated forms, + // self-assignment, and a product operand on either side. + MatrixType expected = a; + result = a; + result += a + perm; + expected += a + permDense; + VERIFY_IS_EQUAL(result, expected); + result -= perm - diag; + expected -= permDense - diagDense; + VERIFY_IS_EQUAL(result, expected); + result -= a - perm; + expected -= a - permDense; + VERIFY_IS_EQUAL(result, expected); + result = a; + result = perm - result; + VERIFY_IS_EQUAL(result, MatrixType(permDense - a)); + VERIFY_IS_APPROX(result = a * a + perm, MatrixType(a * a + permDense)); + VERIFY_IS_APPROX(result = perm + a * a, MatrixType(permDense + a * a)); + VERIFY_IS_APPROX(result = perm - a * a, MatrixType(permDense - a * a)); + VERIFY_IS_APPROX(result = a * a - perm, MatrixType(a * a - permDense)); + + // The inverse (transpose) of a permutation sums the same way. + const MatrixType permInverseDense = permDense.transpose(); + VERIFY_EVALUATION_COUNT(result.noalias() = a + perm.inverse(), 0); + VERIFY_IS_EQUAL(result, MatrixType(a + permInverseDense)); + result = perm.transpose() - a; + VERIFY_IS_EQUAL(result, MatrixType(permInverseDense - a)); + result = perm.inverse() + diag; + VERIFY_IS_EQUAL(result, MatrixType(permInverseDense + diagDense)); + result = diag - perm.inverse(); + VERIFY_IS_EQUAL(result, MatrixType(diagDense - permInverseDense)); + result = a; + result += perm.inverse() + a; + VERIFY_IS_EQUAL(result, MatrixType(a + permInverseDense + a)); + VERIFY_IS_APPROX(result = perm.inverse() - a * a, MatrixType(permInverseDense - a * a)); + VERIFY_IS_APPROX((a - perm.inverse()) * a, (a - permInverseDense) * a); + + // The result is the coefficient-wise one, bit for bit. With dst scaled by 1024, dst + a drops low bits of a, + // so += and -= must add the exact sum at each one; and a diagonal taken from the destination is read before + // it is overwritten. + const MatrixType dst0 = Scalar(1024) * MatrixType::Random(size, size); + const MatrixType dst0Diagonal = dst0.diagonal().asDiagonal(); + result = dst0; + expected = dst0; + result += a + perm; + expected += a + permDense; + VERIFY_IS_EQUAL(result, expected); + result -= perm - a; + expected -= permDense - a; + VERIFY_IS_EQUAL(result, expected); + result += diag - perm; + expected += diagDense - permDense; + VERIFY_IS_EQUAL(result, expected); + result = dst0; + result = result.diagonal().asDiagonal() - perm; + VERIFY_IS_EQUAL(result, MatrixType(dst0Diagonal - permDense)); + result = dst0; + result = perm + result.diagonal().asDiagonal(); + VERIFY_IS_EQUAL(result, MatrixType(permDense + dst0Diagonal)); + + // A row-major destination has the ones of the inverse along its rows. + using RowMajorMatrixType = Matrix<Scalar, Size, Size, RowMajor>; + RowMajorMatrixType rowResult = dst0, rowExpected = dst0; + rowResult += a + perm.inverse(); + rowExpected += a + permInverseDense; + VERIFY_IS_EQUAL(rowResult, rowExpected); + rowResult -= perm.transpose() - diag; + rowExpected -= permInverseDense - diagDense; + VERIFY_IS_EQUAL(rowResult, rowExpected); + rowResult = perm.transpose() - a; + VERIFY_IS_EQUAL(rowResult, RowMajorMatrixType(permInverseDense - a)); + rowResult = dst0; + rowResult = rowResult.diagonal().asDiagonal() + perm.inverse(); + VERIFY_IS_EQUAL(rowResult, RowMajorMatrixType(dst0Diagonal + permInverseDense)); + rowResult = a + perm; + VERIFY_IS_EQUAL(rowResult, RowMajorMatrixType(a + permDense)); +} + +// Operands for which a two-pass assignment (other operand, then the ones) differs from the coefficient-wise sum. +void permutation_dense_sum_assignment_regressions() { + // dst + (a + 1) = 0 for dst = -2^53 and a = 2^53, (dst + a) + 1 = 1. + Matrix<double, 1, 1> dst, a; + dst << -std::ldexp(1.0, 53); + a << std::ldexp(1.0, 53); + PermutationMatrix<1> identity; + identity.setIdentity(); + dst += a + identity; + VERIFY_IS_EQUAL(dst(0, 0), 0.0); + // dst + (a + 1) is defined for dst = INT_MIN and a = -1, dst + a is not. + Matrix<int, 1, 1> r, ai; + r << NumTraits<int>::lowest(); + ai << -1; + r += ai + identity; + VERIFY_IS_EQUAL(r(0, 0), NumTraits<int>::lowest()); + // bool has no negation or subtraction, so a sum must not instantiate them. + Matrix<int, 3, 1> indices; + indices << 2, 0, 1; + const PermutationMatrix<3> perm(indices); + const Matrix<bool, 3, 3> b = Matrix<bool, 3, 3>::Identity(); + const Matrix<bool, 3, 3> expected = b.array() || perm.toDenseMatrix().cast<bool>().array(); + Matrix<bool, 3, 3> bs = b + perm; + VERIFY((bs.array() == expected.array()).all()); + bs = perm + b; + VERIFY((bs.array() == expected.array()).all()); +} + void permutation_inverse_product_temporaries() { PermutationMatrix<Dynamic> lhs(17), rhs(17), result(17); randomPermutationVector(lhs.indices(), lhs.size()); @@ -303,4 +465,14 @@ } CALL_SUBTEST_5(bug890<double>()); CALL_SUBTEST_4(test_aliasing()); + CALL_SUBTEST_10(permutation_dense_sum_assignment_regressions()); + CALL_SUBTEST_10((permutation_dense_sums<float, 1>(1))); + CALL_SUBTEST_10((permutation_dense_sums<double, 3>(3))); + CALL_SUBTEST_10((permutation_dense_sums<std::complex<double>, 4>(4))); + for (int i = 0; i < g_repeat; i++) { + CALL_SUBTEST_10((permutation_dense_sums<float, Dynamic>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); + CALL_SUBTEST_10((permutation_dense_sums<double, Dynamic>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); + CALL_SUBTEST_10( + (permutation_dense_sums<std::complex<double>, Dynamic>(internal::random<int>(1, EIGEN_TEST_MAX_SIZE)))); + } }
diff --git a/test/product_evaluators.cpp b/test/product_evaluators.cpp index f07e5bc..8624ba6 100644 --- a/test/product_evaluators.cpp +++ b/test/product_evaluators.cpp
@@ -4,6 +4,96 @@ #include "main.h" #include <Eigen/Core> +template <typename LhsScalar, typename RhsScalar, int Order> +void outer_product_scalar_types() { + using Scalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType; + using Lhs = Matrix<LhsScalar, Dynamic, 1>; + using Rhs = Matrix<RhsScalar, 1, Dynamic>; + using Mat = Matrix<Scalar, Dynamic, Dynamic, Order>; + STATIC_CHECK((internal::product_type<Lhs, Rhs>::value == OuterProduct)); + for (Index rows : {2, 3, 16, 17}) { + for (Index cols : {2, 3, 16, 17}) { + const Lhs lhs = Lhs::Constant(rows, LhsScalar(2)); + const Rhs rhs = Rhs::Constant(cols, RhsScalar(3)); + Mat storage = Mat::Constant(2 * rows, 2 * cols, Scalar(7)); + Mat expected = storage; + Map<Mat, 0, Stride<Dynamic, 2>> dst(storage.data(), rows, cols, Stride<Dynamic, 2>(2 * storage.outerStride(), 2)); + for (int operation = 0; operation < 3; ++operation) { + storage.setConstant(Scalar(7)); + expected = storage; + if (operation == 0) dst.noalias() = lhs * rhs; + if (operation == 1) dst.noalias() += lhs * rhs; + if (operation == 2) dst.noalias() -= lhs * rhs; + for (Index j = 0; j < cols; ++j) { + for (Index i = 0; i < rows; ++i) { + const Scalar value = lhs(i) * rhs(j); + expected(2 * i, 2 * j) = operation == 0 ? value : operation == 1 ? Scalar(7) + value : Scalar(7) - value; + } + } + VERIFY_IS_EQUAL(storage, expected); + } + } + } +} + +template <typename Real, int Order> +void outer_product_mixed_special_values() { + using Scalar = std::complex<Real>; + using Mat = Matrix<Scalar, Dynamic, Dynamic, Order>; + const Real infinity = NumTraits<Real>::infinity(); + const Real nan = NumTraits<Real>::quiet_NaN(); + const Scalar values[] = {Scalar(infinity, 2), Scalar(2, infinity), Scalar(-infinity, -2), + Scalar(-2, -infinity), Scalar(nan, 2), Scalar(2, nan), + Scalar(Real(0), -Real(0)), Scalar(-Real(0), Real(0))}; + Matrix<Scalar, Dynamic, 1> lhs; + Matrix<Real, 1, Dynamic> rhs; + Mat actual, reversed; + for (Index n : {2, 3, 16, 17}) { + for (const Scalar& value : values) { + for (Real factor : {Real(2), Real(-2)}) { + lhs.setConstant(n, value); + rhs.setConstant(n, factor); + for (int operation = 0; operation < 3; ++operation) { + actual.setConstant(n, n, Scalar(1, 1)); + reversed = actual; + if (operation == 0) { + actual.noalias() = lhs * rhs; + reversed.noalias() = rhs.transpose() * lhs.transpose(); + } + if (operation == 1) { + actual.noalias() += lhs * rhs; + reversed.noalias() += rhs.transpose() * lhs.transpose(); + } + if (operation == 2) { + actual.noalias() -= lhs * rhs; + reversed.noalias() -= rhs.transpose() * lhs.transpose(); + } + const Scalar product(value.real() * factor, value.imag() * factor); + const Scalar expected = operation == 0 ? product + : operation == 1 ? Scalar(1, 1) + product + : Scalar(1, 1) - product; + for (Index j = 0; j < n; ++j) { + for (Index i = 0; i < n; ++i) { + for (int component = 0; component < 2; ++component) { + const Real reference = component == 0 ? expected.real() : expected.imag(); + for (Real result : {component == 0 ? actual(i, j).real() : actual(i, j).imag(), + component == 0 ? reversed(i, j).real() : reversed(i, j).imag()}) { + if ((numext::isnan)(reference)) { + VERIFY((numext::isnan)(result)); + } else { + VERIFY_IS_EQUAL(result, reference); + if (reference == Real(0)) VERIFY_IS_EQUAL((std::signbit)(result), (std::signbit)(reference)); + } + } + } + } + } + } + } + } + } +} + template <typename ProductType, typename Mat> void check_scaled_product(const ProductType& product, const Mat& expected) { Mat actual = product; @@ -129,10 +219,39 @@ Mat aliased = matrix; aliased = alpha * (aliased.template selfadjointView<Mode>() * diagonal.asDiagonal()) + Mat::Zero(); VERIFY_IS_EQUAL(aliased, expected); + check_scaled_product((alpha * matrix.template selfadjointView<Mode>()) * diagonal.asDiagonal(), expected); + check_scaled_product(diagonal.asDiagonal() * (alpha * matrix.template selfadjointView<Mode>()), expectedLeft); + check_scaled_product((alpha * matrix.template selfadjointView<Mode>()) * ownedDiagonal, expected); + check_scaled_product(ownedDiagonal * (alpha * matrix.template selfadjointView<Mode>()), expectedLeft); + const Mat conjugated = matrix.conjugate(); + check_scaled_product((alpha * conjugated.conjugate().template selfadjointView<Mode>()) * diagonal.asDiagonal(), + expected); + check_scaled_product(diagonal.asDiagonal() * (alpha * conjugated.conjugate().template selfadjointView<Mode>()), + expectedLeft); + const Matrix<Scalar, 3, 1> actualDiagonal = + ((alpha * matrix.template selfadjointView<Mode>()) * diagonal.asDiagonal()).diagonal(); + VERIFY_IS_EQUAL(actualDiagonal, expected.diagonal()); + aliased = matrix; + aliased = (alpha * aliased.template selfadjointView<Mode>()) * diagonal.asDiagonal() + Mat::Zero(); + VERIFY_IS_EQUAL(aliased, expected); + aliased = matrix; + aliased = diagonal.asDiagonal() * (alpha * aliased.template selfadjointView<Mode>()) + Mat::Zero(); + VERIFY_IS_EQUAL(aliased, expectedLeft); } EIGEN_DECLARE_TEST(product_evaluators) { for (int repeat = 0; repeat < g_repeat; ++repeat) { + CALL_SUBTEST_4((outer_product_scalar_types<double, double, ColMajor>())); + CALL_SUBTEST_4((outer_product_scalar_types<double, double, RowMajor>())); + CALL_SUBTEST_4((outer_product_scalar_types<std::complex<double>, double, ColMajor>())); + CALL_SUBTEST_4((outer_product_scalar_types<std::complex<double>, double, RowMajor>())); + CALL_SUBTEST_4((outer_product_scalar_types<double, std::complex<double>, ColMajor>())); + CALL_SUBTEST_4((outer_product_scalar_types<double, std::complex<double>, RowMajor>())); + CALL_SUBTEST_4((outer_product_mixed_special_values<float, ColMajor>())); + CALL_SUBTEST_4((outer_product_mixed_special_values<float, RowMajor>())); + CALL_SUBTEST_4((outer_product_mixed_special_values<double, ColMajor>())); + CALL_SUBTEST_4((outer_product_mixed_special_values<double, RowMajor>())); + CALL_SUBTEST_1((scaled_unit_triangular_product<double, UnitLower, ColMajor>())); CALL_SUBTEST_1((scaled_unit_triangular_product<double, UnitLower, RowMajor>())); CALL_SUBTEST_1((scaled_unit_triangular_product<double, UnitUpper, ColMajor>()));
diff --git a/test/product_extra.cpp b/test/product_extra.cpp index 90555e7..ac1f22e 100644 --- a/test/product_extra.cpp +++ b/test/product_extra.cpp
@@ -95,7 +95,7 @@ m1.reverse() * m2.reverse().adjoint()); VERIFY_IS_APPROX(m3.noalias() = map1 * map2.adjoint(), m1.reverse() * m2.reverse().adjoint()); VERIFY_IS_APPROX(map1 * vc2, m1.reverse() * vc2); - VERIFY_IS_APPROX(m1 * mapvc2, m1 * mapvc2); + VERIFY_IS_APPROX(m1 * mapvc2, m1 * vc2.reverse().eval()); VERIFY_IS_APPROX(map1.adjoint() * v1.transpose(), m1.adjoint().reverse() * v1.transpose()); VERIFY_IS_APPROX(m1.adjoint() * mapv1.transpose(), m1.adjoint() * v1.reverse().transpose()); } @@ -656,6 +656,60 @@ (void)PS_d; } +// y += alpha * A * x for every extent of the vectorized dimension from 1 to 12 packets + 2: the GEMV kernels split it +// into passes of up to 8 packets, a final pass of up to 9 packets, and a partial packet (masked where segments exist). +// A is a block of a NaN-padded matrix and x a segment of a NaN-padded vector, so a load past the extent poisons the +// result, and sentinels around y catch a store past it. The reference is an independent long-double loop. +template <typename Scalar, int Order> +void gemv_remainder_sweep() { + using RealScalar = typename NumTraits<Scalar>::Real; + using WideScalar = std::conditional_t<NumTraits<Scalar>::IsComplex, std::complex<long double>, long double>; + using Mat = Matrix<Scalar, Dynamic, Dynamic, Order>; + using Vec = Matrix<Scalar, Dynamic, 1>; + const Index PS = internal::packet_traits<Scalar>::size; + const Scalar nan = Scalar(NumTraits<RealScalar>::quiet_NaN()); + const Scalar sentinel = Scalar(RealScalar(12345)); + const Scalar alpha = internal::random<Scalar>(); + const Index pad = 2 * PS + 1; + // Columns 130 > 128 make the col-major kernel block the columns. + const Index others[] = {1, 2, 3, 5, 9, 17, 130}; + + for (Index vec_extent = 1; vec_extent <= 12 * PS + 2; ++vec_extent) { + for (Index other : others) { + // The vectorized dimension is the rows for col-major and the columns for row-major. + const Index rows = Order == ColMajor ? vec_extent : other; + const Index cols = Order == ColMajor ? other : vec_extent; + Mat big = Mat::Constant(rows + pad, cols + pad, nan); + big.topLeftCorner(rows, cols).setRandom(); + Vec xbig = Vec::Constant(cols + pad, nan); + xbig.head(cols).setRandom(); + Vec ybig = Vec::Constant(rows + 2 * pad, sentinel); + ybig.segment(pad, rows).setRandom(); + const Vec y0 = ybig.segment(pad, rows); + + ybig.segment(pad, rows).noalias() += alpha * big.topLeftCorner(rows, cols) * xbig.head(cols); + + for (Index i = 0; i < rows; ++i) { + WideScalar expected(0); + long double magnitude = 0; + for (Index j = 0; j < cols; ++j) { + const WideScalar term = WideScalar(big(i, j)) * WideScalar(xbig(j)); + expected += term; + magnitude += numext::abs(term); + } + expected = WideScalar(y0(i)) + WideScalar(alpha) * expected; + magnitude = numext::abs(WideScalar(y0(i))) + numext::abs(WideScalar(alpha)) * magnitude; + // Each product and sum rounds once in Scalar, and alpha is applied with one more product and sum. + const long double bound = 4 * (cols + 2) * (long double)NumTraits<RealScalar>::epsilon() * magnitude; + const long double error = numext::abs(WideScalar(ybig(pad + i)) - expected); + VERIFY((numext::isfinite)(error) && error <= bound); + } + VERIFY_IS_CWISE_EQUAL(ybig.head(pad), Vec::Constant(pad, sentinel)); + VERIFY_IS_CWISE_EQUAL(ybig.tail(pad), Vec::Constant(pad, sentinel)); + } + } +} + // Locks the BLAS contract that GEMM/GEMV leave the destination unchanged when // alpha == 0, including under non-finite inputs in A/x/B that would otherwise // taint the result via 0 * Inf = NaN. @@ -789,7 +843,123 @@ VERIFY_IS_EQUAL((t - u * t * u.adjoint()).cwiseAbs().maxCoeff(), RealScalar(0)); } +#if defined(EIGEN_TEST_PART_14) || defined(EIGEN_TEST_PART_ALL) +template <int Order> +void complex_outer_product_overflow() { + using Scalar = std::complex<double>; + using Lhs = Matrix<Scalar, Dynamic, 1>; + using Rhs = Matrix<Scalar, 1, Dynamic>; + using Dst = Matrix<Scalar, Dynamic, Dynamic, Order>; + STATIC_CHECK((internal::product_type<Lhs, Rhs>::value == OuterProduct)); + for (Index n : {4, 17}) { + const Lhs lhs = Lhs::Constant(n, Scalar(1e308, 1)); + const Rhs rhs = Rhs::Constant(n, Scalar(10, 0)); + Dst dst(n, n); + auto check = [&](double sign, double imaginary) { + for (Index j = 0; j < n; ++j) { + for (Index i = 0; i < n; ++i) { + VERIFY((numext::isinf)(dst(i, j).real())); + VERIFY(dst(i, j).real() * sign > 0); + VERIFY_IS_EQUAL(dst(i, j).imag(), imaginary); + } + } + }; + // A redundant multiplication by complex one would introduce 0*infinity into the imaginary part. + dst.noalias() = lhs * rhs; + check(1, 10); + dst.setConstant(Scalar(2, 3)); + dst.noalias() += lhs * rhs; + check(1, 13); + dst.setConstant(Scalar(2, 3)); + dst.noalias() -= lhs * rhs; + check(-1, -7); + } +} + +template <int Order> +void single_term_nullary_product() { + const Vector4d lhs = Vector4d::LinSpaced(1, 4); + const RowVector4d rhs = RowVector4d::LinSpaced(5, 8); + auto check = [&](const auto& left, const auto& right) { + const auto product = left.lazyProduct(right); + Matrix<double, 4, 4, Order> dst = product; + for (Index j = 0; j < 4; ++j) { + for (Index i = 0; i < 4; ++i) { + const double expected = left.coeff(i) * right.coeff(j); + VERIFY_IS_EQUAL(product.coeff(i, j), expected); + VERIFY_IS_EQUAL(dst(i, j), expected); + } + } + }; + check(Vector4d::Ones(), rhs); + check(lhs, RowVector4d::Ones()); + check(Vector4d::Ones(), RowVector4d::Ones()); +} + +namespace outer_product_count { +struct Scalar { + double value; + static int multiplications; + Scalar(double v = 0) : value(v) {} + Scalar operator*(const Scalar& other) const { + ++multiplications; + return Scalar(value * other.value); + } + Scalar operator+(const Scalar& other) const { return Scalar(value + other.value); } + Scalar operator-(const Scalar& other) const { return Scalar(value - other.value); } + Scalar& operator+=(const Scalar& other) { return *this = *this + other; } + Scalar& operator-=(const Scalar& other) { return *this = *this - other; } +}; +int Scalar::multiplications = 0; +} // namespace outer_product_count + +namespace Eigen { +template <> +struct NumTraits<outer_product_count::Scalar> : GenericNumTraits<outer_product_count::Scalar> { + static constexpr bool RequireInitialization = true; +}; +} // namespace Eigen + +template <int Order> +void small_outer_product_unscaled() { + using Scalar = outer_product_count::Scalar; + using Lhs = Matrix<Scalar, Dynamic, 1>; + using Rhs = Matrix<Scalar, 1, Dynamic>; + using Dst = Matrix<Scalar, Dynamic, Dynamic, Order>; + STATIC_CHECK((internal::product_type<Lhs, Rhs>::value == OuterProduct)); + for (Index n : {1, 8, 16, 17}) { + const Lhs lhs = Lhs::Constant(n, Scalar(2)); + const Rhs rhs = Rhs::Constant(n, Scalar(3)); + Dst dst(n, n); + auto check = [&](double expected) { + VERIFY_IS_EQUAL(Scalar::multiplications, n * n); + for (Index j = 0; j < n; ++j) + for (Index i = 0; i < n; ++i) VERIFY_IS_EQUAL(dst(i, j).value, expected); + }; + Scalar::multiplications = 0; + dst.noalias() = lhs * rhs; + check(6); + Scalar::multiplications = 0; + dst.noalias() += lhs * rhs; + check(12); + Scalar::multiplications = 0; + dst.noalias() -= lhs * rhs; + check(6); + internal::generic_product_impl<Lhs, Rhs>::scaleAndAddTo(dst, lhs, rhs, Scalar(2)); + for (Index j = 0; j < n; ++j) + for (Index i = 0; i < n; ++i) VERIFY_IS_EQUAL(dst(i, j).value, 18.0); + } +} + +#endif + EIGEN_DECLARE_TEST(product_extra) { + CALL_SUBTEST_14(complex_outer_product_overflow<ColMajor>()); + CALL_SUBTEST_14(complex_outer_product_overflow<RowMajor>()); + CALL_SUBTEST_14(single_term_nullary_product<ColMajor>()); + CALL_SUBTEST_14(single_term_nullary_product<RowMajor>()); + CALL_SUBTEST_14(small_outer_product_unscaled<ColMajor>()); + CALL_SUBTEST_14(small_outer_product_unscaled<RowMajor>()); CALL_SUBTEST_13((complex_gemm_scalar_accumulation<float, ColMajor>())); CALL_SUBTEST_13((complex_gemm_scalar_accumulation<float, RowMajor>())); CALL_SUBTEST_13((complex_gemm_scalar_accumulation<double, ColMajor>())); @@ -831,4 +1001,13 @@ CALL_SUBTEST_12(alpha_zero_skips_kernel<double>()); CALL_SUBTEST_12(alpha_zero_skips_kernel<std::complex<float> >()); CALL_SUBTEST_12(alpha_zero_skips_kernel<std::complex<double> >()); + + CALL_SUBTEST_15((gemv_remainder_sweep<float, ColMajor>())); + CALL_SUBTEST_15((gemv_remainder_sweep<float, RowMajor>())); + CALL_SUBTEST_15((gemv_remainder_sweep<double, ColMajor>())); + CALL_SUBTEST_15((gemv_remainder_sweep<double, RowMajor>())); + CALL_SUBTEST_16((gemv_remainder_sweep<std::complex<float>, ColMajor>())); + CALL_SUBTEST_16((gemv_remainder_sweep<std::complex<float>, RowMajor>())); + CALL_SUBTEST_16((gemv_remainder_sweep<std::complex<double>, ColMajor>())); + CALL_SUBTEST_16((gemv_remainder_sweep<std::complex<double>, RowMajor>())); }
diff --git a/test/product_large.cpp b/test/product_large.cpp index 1a5ceab..599a2d6 100644 --- a/test/product_large.cpp +++ b/test/product_large.cpp
@@ -11,10 +11,11 @@ #include "product.h" #include <Eigen/LU> -#if (defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX)) && defined(__unix__) +#if defined(__unix__) #include <sys/mman.h> #include <unistd.h> +#if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) template <int> void gemv_bfloat16_strided_tail() { using MatrixType = Matrix<bfloat16, Dynamic, Dynamic, RowMajor>; @@ -44,6 +45,97 @@ } #endif +#if (defined(EIGEN_VECTORIZE_AVX512) && EIGEN_USE_AVX512_GEMM_KERNELS) || defined(EIGEN_VECTORIZE_VSX) +// Place packed A, packed B, and C right before PROT_NONE guard pages to catch +// any out-of-bounds lookahead or tail over-read in GEBP kernels. +template <typename Scalar, int ResIncr = Dynamic> +void gebp_guard_page_tail() { + struct GuardedArena { + long page_size; + std::size_t data_pages; + std::size_t map_bytes; + char* raw; + + explicit GuardedArena(std::size_t max_bytes) { + page_size = sysconf(_SC_PAGESIZE); + VERIFY(page_size > 0); + data_pages = (max_bytes + page_size - 1) / page_size; + if (data_pages == 0) data_pages = 1; + map_bytes = (data_pages + 1) * page_size; + raw = static_cast<char*>(mmap(nullptr, map_bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + VERIFY(raw != MAP_FAILED); + VERIFY_IS_EQUAL(mprotect(raw + data_pages * page_size, page_size, PROT_NONE), 0); + } + ~GuardedArena() { VERIFY_IS_EQUAL(munmap(raw, map_bytes), 0); } + + Scalar* place_at_end(std::size_t n_elems) { + VERIFY(n_elems * sizeof(Scalar) <= data_pages * page_size); + return reinterpret_cast<Scalar*>(raw + data_pages * page_size) - n_elems; + } + }; + + using Traits = internal::gebp_traits<Scalar, Scalar>; + using ResMapper = internal::blas_data_mapper<Scalar, Index, ColMajor, Unaligned, ResIncr>; + using LhsMapper = internal::const_blas_data_mapper<Scalar, Index, ColMajor>; + using RhsMapper = internal::const_blas_data_mapper<Scalar, Index, ColMajor>; + internal::gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, + ColMajor> + pack_lhs; + internal::gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, ColMajor> pack_rhs; + internal::gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, false, false> gebp; + + GuardedArena arenaA(64 * 64 * sizeof(Scalar)); + GuardedArena arenaB(64 * 64 * sizeof(Scalar)); + GuardedArena arenaC(64 * 64 * 2 * sizeof(Scalar)); + + for (Index m : {0, 1, 2, 3, 4, 8, 16, 24, 32, 48, 53}) { + for (Index n : {0, 1, 2, 3, 4, 5, 8, 9, 13}) { + for (Index k : {0, 1, 2, 3, 4, 5, 8, 9, 17}) { + Matrix<Scalar, Dynamic, Dynamic, ColMajor> A = Matrix<Scalar, Dynamic, Dynamic, ColMajor>::Random(m, k); + Matrix<Scalar, Dynamic, Dynamic, ColMajor> B = Matrix<Scalar, Dynamic, Dynamic, ColMajor>::Random(k, n); + Matrix<Scalar, Dynamic, Dynamic, ColMajor> C0 = Matrix<Scalar, Dynamic, Dynamic, ColMajor>::Random(m, n); + + Scalar* blockA = arenaA.place_at_end(m * k); + Scalar* blockB = arenaB.place_at_end(k * n); + if (m > 0 && k > 0) { + Matrix<Scalar, Dynamic, 1> packedA(m * k); + pack_lhs(packedA.data(), LhsMapper(A.data(), A.outerStride()), k, m); + std::copy_n(packedA.data(), m * k, blockA); + } + if (k > 0 && n > 0) { + pack_rhs(blockB, RhsMapper(B.data(), B.outerStride()), k, n); + } + + for (Index inc : {1, 2}) { + if (ResIncr == 1 && inc != 1) continue; + for (Scalar alpha : {Scalar(1), Scalar(2)}) { + Index ldc = m * inc; + Scalar* C = arenaC.place_at_end(ldc * n); + for (Index j = 0; j < n; ++j) { + for (Index i = 0; i < m; ++i) { + C[j * ldc + i * inc] = C0(i, j); + } + } + + ResMapper res(C, ldc, inc); + gebp(res, blockA, blockB, m, k, n, alpha, -1, -1, 0, 0); + + if (m > 0 && n > 0) { + Matrix<Scalar, Dynamic, Dynamic, ColMajor> Cref = C0; + if (k > 0) Cref.noalias() += alpha * (A * B); + Map<const Matrix<Scalar, Dynamic, Dynamic, ColMajor>, 0, InnerStride<Dynamic>> Cmap( + C, m, n, InnerStride<Dynamic>(inc)); + VERIFY_IS_APPROX(Cmap, Cref); + } + } + } + } + } + } +} +#endif +#endif + template <typename T> void test_aliasing() { int rows = internal::random<int>(1, 12); @@ -375,6 +467,17 @@ CALL_SUBTEST_6(gemv_small_cols_systematic<0>()); CALL_SUBTEST_6(gemv_rowmajor_large_stride_varied_rows<0>()); CALL_SUBTEST_6(product_extreme_aspect_ratios<0>()); +#if defined(__unix__) && \ + ((defined(EIGEN_VECTORIZE_AVX512) && EIGEN_USE_AVX512_GEMM_KERNELS) || defined(EIGEN_VECTORIZE_VSX)) + CALL_SUBTEST_6(gebp_guard_page_tail<float>()); + CALL_SUBTEST_6(gebp_guard_page_tail<double>()); + CALL_SUBTEST_6(gebp_guard_page_tail<std::complex<float>>()); + CALL_SUBTEST_6(gebp_guard_page_tail<std::complex<double>>()); + CALL_SUBTEST_6((gebp_guard_page_tail<float, 1>())); + CALL_SUBTEST_6((gebp_guard_page_tail<double, 1>())); + CALL_SUBTEST_6((gebp_guard_page_tail<std::complex<float>, 1>())); + CALL_SUBTEST_6((gebp_guard_page_tail<std::complex<double>, 1>())); +#endif // Regression test for bug 714: #if defined EIGEN_HAS_OPENMP
diff --git a/test/product_small.cpp b/test/product_small.cpp index 054c637..21396a7 100644 --- a/test/product_small.cpp +++ b/test/product_small.cpp
@@ -262,6 +262,36 @@ VERIFY_IS_APPROX(res, A * b); } +template <typename MatrixType> +void fixed_size_product_assignments() { + MatrixType A = MatrixType::Random(); + MatrixType B = MatrixType::Random(); + MatrixType C = MatrixType::Random(); + MatrixType ref = C; + + C = A * B; + ref = A.lazyProduct(B); + VERIFY_IS_APPROX(C, ref); + + C.setRandom(); + ref = C; + C.noalias() += A * B; + ref.noalias() += A.lazyProduct(B); + VERIFY_IS_APPROX(C, ref); + + C.setRandom(); + ref = C; + C.noalias() -= A * B; + ref.noalias() -= A.lazyProduct(B); + VERIFY_IS_APPROX(C, ref); + + C.setRandom(); + ref = C; + C.noalias() += (2.0 * A) * (B * 3.0); + ref.noalias() += 6.0 * A.lazyProduct(B); + VERIFY_IS_APPROX(C, ref); +} + template <int> void product_small_regressions() { { @@ -307,36 +337,30 @@ constexpr int kGemm = (internal::product_type<Matrix10, Matrix10>::FixedSizeThreshold + 2) / 3; typedef Eigen::Matrix<double, kGemm - 1, kGemm - 1> MatrixBelow; typedef Eigen::Matrix<double, kGemm, kGemm> MatrixAt; - VERIFY((internal::product_type<Matrix10, Matrix10>::value == CoeffBasedProductMode)); VERIFY((internal::product_type<MatrixBelow, MatrixBelow>::value == CoeffBasedProductMode)); VERIFY((internal::product_type<MatrixAt, MatrixAt>::value == GemmProduct)); - Matrix10 A = Matrix10::Random(); - Matrix10 B = Matrix10::Random(); - Matrix10 C = Matrix10::Random(); - Matrix10 ref = C; + fixed_size_product_assignments<MatrixBelow>(); + fixed_size_product_assignments<MatrixAt>(); - C = A * B; - ref = A.lazyProduct(B); - VERIFY_IS_APPROX(C, ref); - - C.setRandom(); - ref = C; - C.noalias() += A * B; - ref.noalias() += A.lazyProduct(B); - VERIFY_IS_APPROX(C, ref); - - C.setRandom(); - ref = C; - C.noalias() -= A * B; - ref.noalias() -= A.lazyProduct(B); - VERIFY_IS_APPROX(C, ref); - - C.setRandom(); - ref = C; - C.noalias() += (2.0 * A) * (B * 3.0); - ref.noalias() += 6.0 * A.lazyProduct(B); - VERIFY_IS_APPROX(C, ref); + using Prod3d = internal::product_evaluator<Product<Matrix3d, Matrix3d, LazyProduct>, CoeffBasedProductMode, + DenseShape, DenseShape>; + using Prod4d = internal::product_evaluator<Product<Matrix4d, Matrix4d, LazyProduct>, CoeffBasedProductMode, + DenseShape, DenseShape>; + using Add3d = internal::add_assign_op<double, double>; + using RowBlock3d = Block<Matrix3d, 1, 3>; + static_assert(!internal::product_packet_cascade_traits<Prod4d, Matrix4d, Add3d>::Enable, + "exact packet multiple keeps the generic path"); + static_assert(!internal::product_packet_cascade_traits<Prod3d, RowBlock3d, Add3d>::Enable, + "non-unit inner stride must not use the contiguous packet cascade"); +#if defined(EIGEN_VECTORIZE) && EIGEN_UNALIGNED_VECTORIZE + if (internal::unpacket_traits<internal::find_largest_packet<double, 3>::type>::size == 2) { + VERIFY((internal::product_packet_cascade_traits<Prod3d, Matrix3d, Add3d>::Enable)); + } +#else + static_assert(!internal::product_packet_cascade_traits<Prod3d, Matrix3d, Add3d>::Enable, + "cascade requires ActualPacketAccessBit and EIGEN_UNALIGNED_VECTORIZE"); +#endif } }
diff --git a/test/product_sme.cpp b/test/product_sme.cpp index f676904..d48423c 100644 --- a/test/product_sme.cpp +++ b/test/product_sme.cpp
@@ -18,6 +18,8 @@ #include "product.h" +#include <cfenv> + // Without the right -march flags, __ARM_FEATURE_SME is undefined and // EIGEN_VECTORIZE_SME never fires - the test would silently compile // against the NEON GEBP kernel and pass, making this a useless no-op. @@ -72,9 +74,11 @@ // the literal-plus-Index bounds the packers pass. Name every spelling Index // takes, mix the two argument types, and check the predicate so the traits' // widening cannot change which lanes are active. +// EIGEN_DONT_INLINE here and below: GCC (14.1 through trunk) inlines a __arm_locally_streaming function into a +// non-streaming caller without its mode switch, running the body as non-streaming SVE at that vector length. template <typename Scalar, typename Begin, typename End = Begin> -__arm_locally_streaming static bool sme_whilelt_covers_first_lane_only() { - using Traits = internal::sme_traits<Scalar>; +EIGEN_DONT_INLINE __arm_locally_streaming static bool sme_whilelt_covers_first_lane_only() { + using Traits = internal::sme_packet_traits<Scalar>; const svbool_t pg = Traits::whilelt(Begin(0), End(1)); return svptest_first(Traits::ptrue(), pg) && svcntp_b8(Traits::ptrue(), pg) == 1; } @@ -88,6 +92,55 @@ VERIFY((sme_whilelt_covers_first_lane_only<Scalar, int, Index>())); } +// Column j of out (n rows, one streaming vector) receives the j-th operation's result. +template <typename Scalar> +EIGEN_DONT_INLINE __arm_locally_streaming static Scalar sme_packet_ops(Scalar* out, const Scalar* a, const Scalar* b, + const Scalar* c, Index active) { + using Traits = internal::sme_packet_traits<Scalar>; + using Vec = typename Traits::type; + const svbool_t all = Traits::ptrue(); + const svbool_t pg = Traits::whilelt(0, active); + const Index n = Traits::size(); + const Vec va = internal::ploadu(all, a), vb = internal::ploadu(all, b), vc = internal::ploadu(all, c); + internal::pstoreu(all, out, internal::pmadd(all, va, vb, vc)); + internal::pstoreu(all, out + n, internal::pnmadd(all, va, vb, vc)); + internal::pstoreu(all, out + 2 * n, internal::pmadd_m(pg, va, vb, vc)); + internal::pstoreu(all, out + 3 * n, internal::padd(all, va, vb)); + internal::pstoreu(all, out + 4 * n, internal::pmul(all, va, vb)); + internal::pstoreu(all, out + 5 * n, internal::pnegate(all, va)); + internal::pstoreu(pg, out + 6 * n, internal::pset1<Vec>(Scalar(7))); + return internal::predux(pg, va); +} + +// Operand order and predication of the packet layer, on small integers so every result is exact. +// a_i = i + 2, b_i = -(i + 3), c_i = 2i + 5 are pairwise distinct and never +-1, so swapping two +// operands of pmadd or pnmadd changes every lane: ab + c - (ac + b) = (b - c)(a - 1), and so on. +template <typename Scalar> +static void test_packet_ops() { + using Vec = Matrix<Scalar, Dynamic, 1>; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + const Index n = internal::sme_packet_traits<Scalar>::size(); + const Vec a = Vec::LinSpaced(n, Scalar(2), Scalar(n + 1)); + const Vec b = Vec::LinSpaced(n, Scalar(-3), Scalar(-(n + 2))); + const Vec c = Vec::LinSpaced(n, Scalar(5), Scalar(2 * n + 3)); + for (Index active = 0; active <= n; ++active) { + Mat out = Mat::Constant(n, 7, Scalar(-1)); + const Scalar sum = sme_packet_ops(out.data(), a.data(), b.data(), c.data(), active); + Mat ref(n, 7); + ref.col(0) = a.cwiseProduct(b) + c; + ref.col(1) = c - a.cwiseProduct(b); + ref.col(2) = c; + ref.col(2).head(active) = ref.col(0).head(active); + ref.col(3) = a + b; + ref.col(4) = a.cwiseProduct(b); + ref.col(5) = -a; + ref.col(6).setConstant(Scalar(-1)); + ref.col(6).head(active).setConstant(Scalar(7)); + VERIFY_IS_EQUAL(out, ref); + VERIFY_IS_EQUAL(sum, a.head(active).sum()); + } +} + // Write one element into a packed panel of width w, in the layout the SME // kernel reads: a real scalar lands at dst[k*w + r], a complex one splits into // the depth step's real and imaginary halves. @@ -198,6 +251,48 @@ C.setZero(); C.noalias() += alpha * (A.adjoint() * B); VERIFY_IS_APPROX(SmeColMajorMat<Scalar>(C), SmeColMajorMat<Scalar>(alpha * Aa.lazyProduct(Bp))); + + // A deep rectangular product reaches GEMM even for half-tile output widths. + const LhsMat deepA = LhsMat::Random(n, 65); + const RhsMat deepB = RhsMat::Random(65, n); + const ResMat initial = ResMat::Random(n, n); + const SmeColMajorMat<Scalar> reference = deepA.lazyProduct(deepB); + C = initial; + C.noalias() += deepA * deepB; + VERIFY_IS_APPROX(SmeColMajorMat<Scalar>(C), SmeColMajorMat<Scalar>(initial + reference)); + C = initial; + C.noalias() += alpha * (deepA * deepB); + VERIFY_IS_APPROX(SmeColMajorMat<Scalar>(C), SmeColMajorMat<Scalar>(initial + alpha * reference)); + // A real alpha takes the complex store's fused path: -1 from -=, and a + // non-unit real scale. + const Scalar real_alpha(typename NumTraits<Scalar>::Real(1.375)); + C = initial; + C.noalias() -= deepA * deepB; + VERIFY_IS_APPROX(SmeColMajorMat<Scalar>(C), SmeColMajorMat<Scalar>(initial - reference)); + C = initial; + C.noalias() += real_alpha * (deepA * deepB); + VERIFY_IS_APPROX(SmeColMajorMat<Scalar>(C), SmeColMajorMat<Scalar>(initial + real_alpha * reference)); + +#ifdef EIGEN_VECTORIZE_SME_F64F64 + // RowMajor GEMM normally transposes the computation. Call the packed kernel + // too, so RowMajor C actually reaches its horizontal ZA slice store. + if (n <= sme_tile<Scalar>()) { + std::vector<Scalar> packedA(n * 65), packedB(n * 65); + for (Index k = 0; k < 65; ++k) { + for (Index i = 0; i < n; ++i) { + set_packed(packedA.data(), Index(n), k, i, deepA(i, k)); + set_packed(packedB.data(), Index(n), k, i, deepB(k, i)); + } + } + for (Scalar scale : {Scalar(1), alpha, Scalar(-1), real_alpha}) { + C = initial; + internal::sme_gebp_dispatch<Scalar, false, false>::run( + C.data(), Index(ResMat::IsRowMajor ? n : 1), Index(ResMat::IsRowMajor ? 1 : n), packedA.data(), + packedB.data(), Index(n), Index(65), Index(n), scale, Index(65), Index(65), Index(0), Index(0)); + VERIFY_IS_APPROX(SmeColMajorMat<Scalar>(C), SmeColMajorMat<Scalar>(initial + scale * reference)); + } + } +#endif } // Exercise the kc split path just above the SME blocking heuristic's depth cap @@ -217,6 +312,278 @@ VERIFY_IS_APPROX(C, c_before + (A.lazyProduct(B)).eval()); } +// LHS read in place: strides on either side of the 4 KB-multiple rule (packed versus in place), an odd stride +// and base offset, a sub-block at a row offset, and a depth tail, each checked against lazyProduct. +template <typename Scalar> +static void test_direct_lhs() { + const Index kPage = 4096 / Index(sizeof(Scalar)); + const Index rows = 97, depth = 101, cols = 70; + const Index strides[] = {rows + 32 + 1, kPage, kPage + 16, 2 * kPage + 16}; + for (Index lda : strides) { + for (Index offset = 0; offset <= 1; ++offset) { + SmeVector<Scalar> storage = SmeVector<Scalar>::Zero(lda * depth + 1); + Map<SmeColMajorMat<Scalar>, 0, OuterStride<>> A(storage.data() + offset, rows + 32, depth, OuterStride<>(lda)); + A.setRandom(); + const SmeColMajorMat<Scalar> B = SmeColMajorMat<Scalar>::Random(depth, cols); + const SmeColMajorMat<Scalar> C0 = SmeColMajorMat<Scalar>::Random(rows + 32, cols); + SmeColMajorMat<Scalar> C = C0; + C.noalias() += A * B; + VERIFY_IS_APPROX(C, C0 + A.lazyProduct(B)); + SmeColMajorMat<Scalar> D = C0.middleRows(3, rows); + D.noalias() += A.middleRows(3, rows) * B; + VERIFY_IS_APPROX(D, C0.middleRows(3, rows) + A.middleRows(3, rows).lazyProduct(B)); + } + } +#ifdef EIGEN_VECTORIZE_SME + const int units = nbSmeUnits(); + VERIFY(units >= 0); + setNbSmeUnits(1); + VERIFY_IS_EQUAL(nbSmeUnits(), 1); + setNbSmeUnits(units); + VERIFY_IS_EQUAL(nbSmeUnits(), units); +#endif + // An LHS packed once for all column blocks, where the first block reads it in place and a narrower last one, whose + // panel spans more than EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES, packs. + { + const Index big = Index(EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES) / (2048 * Index(sizeof(Scalar))) + 2; + const SmeColMajorMat<Scalar> Abig = SmeColMajorMat<Scalar>::Random(big, 2048); + const auto A = Abig.topRows(256); + const SmeColMajorMat<Scalar> B = SmeColMajorMat<Scalar>::Random(2048, 532); + SmeColMajorMat<Scalar> C(256, 532); + C.noalias() = A * B; + VERIFY_IS_APPROX(C, A.lazyProduct(B)); + } +} + +// Small K against large M and N (NEON depths and just past them), and a RHS of one panel read in place: up to +// half a panel wide always, up to a full panel within the span limit, with 4 KB-multiple strides. +template <typename Scalar> +static void test_small_k_and_single_panel_rhs() { + const Index MR = sme_mr<Scalar>(), NR = sme_nr<Scalar>(); + for (Index k : {1, 2, 3, 5, 8, 17, 24, 25, 40}) { + for (Index mn : {Index(130), 4 * MR + 3}) { + const SmeColMajorMat<Scalar> A = SmeColMajorMat<Scalar>::Random(mn, k), B = SmeColMajorMat<Scalar>::Random(k, mn); + SmeColMajorMat<Scalar> C = SmeColMajorMat<Scalar>::Random(mn, mn); + const SmeColMajorMat<Scalar> C0 = C; + C.noalias() += A * B; + VERIFY_IS_APPROX(C, C0 + A.lazyProduct(B)); + } + } + const Index page = 4096 / Index(sizeof(Scalar)); + for (Index n : {Index(1), NR / 2 - 1, NR / 2, NR / 2 + 1, NR}) { + for (Index lda : {5 * MR + 3, page, 2 * page + 1}) { + for (Index depth : {Index(40), Index(301)}) { + const Index rows = numext::mini(lda, 5 * MR + 3); + SmeVector<Scalar> storage = SmeVector<Scalar>::Zero(lda * depth); + Map<SmeColMajorMat<Scalar>, 0, OuterStride<>> A(storage.data(), rows, depth, OuterStride<>(lda)); + A.setRandom(); + const SmeColMajorMat<Scalar> B = SmeColMajorMat<Scalar>::Random(depth, n); + SmeColMajorMat<Scalar> C = SmeColMajorMat<Scalar>::Random(rows, n); + const SmeColMajorMat<Scalar> C0 = C; + C.noalias() += A * B; + VERIFY_IS_APPROX(C, C0 + A.lazyProduct(B)); + } + } + } + // A single-panel RHS whose depth block spans more than EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES packs the LHS. + const Index kc = Index(EIGEN_SME_MAX_KC) * Index(sizeof(float)) / Index(sizeof(Scalar)); + const Index wide = page * Index(std::size_t(EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES) / (std::size_t(kc) * 4096) + 1); + SmeVector<Scalar> storage = SmeVector<Scalar>::Zero(wide * kc); + Map<SmeColMajorMat<Scalar>, 0, OuterStride<>> A(storage.data(), 2 * MR + 5, kc, OuterStride<>(wide)); + A.setRandom(); + const SmeColMajorMat<Scalar> B = SmeColMajorMat<Scalar>::Random(kc, NR); + SmeColMajorMat<Scalar> C = SmeColMajorMat<Scalar>::Zero(2 * MR + 5, NR); + C.noalias() += A * B; + VERIFY_IS_APPROX(C, A.lazyProduct(B)); +} + +// Disjoint parts per SME unit (2 and 3 units) against a serial product. product_threaded covers the thread pool; +// this reaches the OpenMP branch in an EIGEN_TEST_OPENMP build and runs serially otherwise. +template <typename Scalar> +static void test_disjoint_parts() { + using Col = SmeColMajorMat<Scalar>; + using Row = SmeRowMajorMat<Scalar>; + const int saved = nbSmeUnits(); + const int shapes[][3] = {{301, 259, 131}, {200, 40, 1024}, {20, 700, 300}}; + for (int units : {2, 3}) { + setNbSmeUnits(units); + for (const auto& sh : shapes) { + const Col A = Col::Random(sh[0], sh[2]), B = Col::Random(sh[2], sh[1]); + const Col ref = A.lazyProduct(B); + Col C(sh[0], sh[1]); + C.noalias() = A * B; + VERIFY_IS_APPROX(C, ref); + Row R(sh[0], sh[1]); + R.noalias() = A * B; + VERIFY_IS_APPROX(Col(R), ref); + Col big = Col::Random(sh[0] + 5, sh[1] + 3); + Col expected = big; + expected.block(2, 1, sh[0], sh[1]) += Scalar(1.5) * ref; + big.block(2, 1, sh[0], sh[1]).noalias() += Scalar(1.5) * A * B; + VERIFY_IS_APPROX(big, expected); + } + } + setNbSmeUnits(saved); +} + +// Streaming-mode entry and exit set every FP exception flag: products on each SME path, with exact results, must leave +// the caller's flags as they were. +template <typename Scalar, typename Product> +static void verify_fp_flags_kept(Product product) { +#if defined(EIGEN_HAS_OPENMP) || defined(EIGEN_GEMM_THREADPOOL) + // A threaded build's parallelize_gemm sizes the thread count in floating point, which can raise FE_INEXACT. + const int checked = FE_ALL_EXCEPT & ~FE_INEXACT; +#else + const int checked = FE_ALL_EXCEPT; +#endif + std::feclearexcept(FE_ALL_EXCEPT); + product(); + VERIFY_IS_EQUAL(std::fetestexcept(checked), 0); + std::feraiseexcept(FE_INVALID); + product(); + VERIFY_IS_EQUAL(std::fetestexcept(checked), FE_INVALID); + std::feclearexcept(FE_ALL_EXCEPT); +} + +template <typename Scalar> +static void test_fp_flags() { + using Col = SmeColMajorMat<Scalar>; + using Row = SmeRowMajorMat<Scalar>; + const Index MR = sme_mr<Scalar>(), NR = sme_nr<Scalar>(); + const Index m = 4 * MR + 3, n = 3 * NR + 5, k = 131; + // Small integers keep every product exact, so no flag is legitimately raised. + const Col A = Col::Random(m, k).unaryExpr([](const Scalar& x) { return Scalar(numext::round(numext::real(x) * 4)); }); + const Col B = Col::Random(k, n).unaryExpr([](const Scalar& x) { return Scalar(numext::round(numext::real(x) * 4)); }); + const Col S = Col::Identity(m, m) * Scalar(2) + Col::Ones(m, m), O = Col::Ones(m, n); + // I plus a subdiagonal of ones: the solution of L x = 1 is 1, 0, 1, 0, ..., exact. + Col L = Col::Identity(m, m); + L.diagonal(-1).setOnes(); + const Col Tall = Col::Ones(1024, k), Thin = Col::Ones(k, NR / 2); + Col C(m, n), D(1024, NR / 2); + Row R(m, n); + verify_fp_flags_kept<Scalar>([&] { C.noalias() = A * B; }); + verify_fp_flags_kept<Scalar>([&] { C.noalias() += A.conjugate() * B; }); + verify_fp_flags_kept<Scalar>([&] { R.noalias() = A * B; }); + verify_fp_flags_kept<Scalar>([&] { R.noalias() = Row(A) * Row(B); }); + verify_fp_flags_kept<Scalar>([&] { D.noalias() = Tall * Thin; }); + verify_fp_flags_kept<Scalar>([&] { C.noalias() = S.template selfadjointView<Lower>() * O; }); + verify_fp_flags_kept<Scalar>([&] { C.noalias() = S.template triangularView<Upper>() * O; }); + verify_fp_flags_kept<Scalar>([&] { + Col X = O; + L.template triangularView<Lower>().solveInPlace(X); + }); +} + +// Tiny results (the NEON kernel and both sides of its routing), every storage order of A, B and C. +template <typename Scalar, int AO, int BO, int CO> +static void test_tiny_results_order() { + using MA = Matrix<Scalar, Dynamic, Dynamic, AO>; + using MB = Matrix<Scalar, Dynamic, Dynamic, BO>; + using MC = Matrix<Scalar, Dynamic, Dynamic, CO>; + const Index ps = Index(16 / sizeof(Scalar)); + for (Index m : {Index(1), Index(2), Index(3), ps, ps + 1, 2 * ps, 2 * ps + 1}) + for (Index n : {1, 2, 3, 5, 8, 9}) + for (Index k : {1, 5, 6, 7, 15, 16, 17, 64, 2049}) { + const MA A = MA::Random(m, k); + const MB B = MB::Random(k, n); + const MC ref = A.lazyProduct(B); + MC C = MC::Random(m, n); + const MC C0 = C; + C.noalias() = A * B; + VERIFY_IS_APPROX(C, ref); + C = C0; + C.noalias() += A * B; + VERIFY_IS_APPROX(C, C0 + ref); + C = C0; + C.noalias() -= Scalar(2) * A * B; + VERIFY_IS_APPROX(C, C0 - Scalar(2) * ref); + const MA At = A.transpose(); + C.noalias() = At.transpose() * B; + VERIFY_IS_APPROX(C, ref); + } +} + +template <typename Scalar> +static void test_tiny_results() { + test_tiny_results_order<Scalar, ColMajor, ColMajor, ColMajor>(); + test_tiny_results_order<Scalar, RowMajor, ColMajor, ColMajor>(); + test_tiny_results_order<Scalar, ColMajor, RowMajor, ColMajor>(); + test_tiny_results_order<Scalar, RowMajor, RowMajor, ColMajor>(); + test_tiny_results_order<Scalar, ColMajor, ColMajor, RowMajor>(); + test_tiny_results_order<Scalar, RowMajor, ColMajor, RowMajor>(); + test_tiny_results_order<Scalar, ColMajor, RowMajor, RowMajor>(); + test_tiny_results_order<Scalar, RowMajor, RowMajor, RowMajor>(); + // Sub-blocks of larger operands, a result with an inner stride, and a Hankel view (outer stride 1 < rows): the + // kernel reads only the rows and columns of the result. + for (Index m : {Index(3), Index(16 / sizeof(Scalar)) + 1}) + for (Index n : {3, 7}) + for (Index k : {16, 19}) { + const SmeColMajorMat<Scalar> Abig = SmeColMajorMat<Scalar>::Random(m + 5, k + 2); + const SmeRowMajorMat<Scalar> Bbig = SmeRowMajorMat<Scalar>::Random(k + 3, n + 4); + const auto A = Abig.block(2, 1, m, k); + const auto B = Bbig.block(1, 2, k, n); + SmeColMajorMat<Scalar> Cbig = SmeColMajorMat<Scalar>::Random(2 * m, n); + const SmeColMajorMat<Scalar> Cbig0 = Cbig; + Map<SmeColMajorMat<Scalar>, 0, Stride<Dynamic, 2>> C(Cbig.data(), m, n, Stride<Dynamic, 2>(2 * m, 2)); + C.noalias() += Scalar(3) * A * B; + SmeColMajorMat<Scalar> ref = Cbig0; + Map<SmeColMajorMat<Scalar>, 0, Stride<Dynamic, 2>>(ref.data(), m, n, Stride<Dynamic, 2>(2 * m, 2)) += + Scalar(3) * A.lazyProduct(B); + VERIFY_IS_APPROX(Cbig, ref); + std::vector<Scalar> h(std::size_t(k - 1 + m)), g(std::size_t(k - 1 + n)); + for (auto& x : h) x = internal::random<Scalar>(); + for (auto& x : g) x = internal::random<Scalar>(); + Map<const SmeColMajorMat<Scalar>, 0, OuterStride<>> H(h.data(), m, k, OuterStride<>(1)); + Map<const SmeRowMajorMat<Scalar>, 0, OuterStride<>> G(g.data(), k, n, OuterStride<>(1)); + SmeColMajorMat<Scalar> D(m, n); + D.noalias() = H * G; + VERIFY_IS_APPROX(D, H.lazyProduct(G)); + } + // Operands packed tight at the end of their buffer, so a read past the last row or column leaves it. + const Index ps = Index(16 / sizeof(Scalar)); + for (Index m : {Index(2), Index(3), ps + 1, 2 * ps}) + for (Index n : {2, 3, 5, 7}) + for (Index k : {16, 17, 18, 19}) { + std::vector<Scalar> a(std::size_t(m * k)), b(std::size_t(k * n)); + Map<SmeColMajorMat<Scalar>> A(a.data(), m, k); + Map<SmeRowMajorMat<Scalar>> B(b.data(), k, n); + A.setRandom(); + B.setRandom(); + SmeColMajorMat<Scalar> C = SmeColMajorMat<Scalar>::Zero(m, n); + C.noalias() += A * B; + VERIFY_IS_APPROX(C, A.lazyProduct(B)); + Map<SmeRowMajorMat<Scalar>> Ar(a.data(), m, k); + Map<SmeColMajorMat<Scalar>> Bc(b.data(), k, n); + C.noalias() = Ar * Bc; + VERIFY_IS_APPROX(C, Ar.lazyProduct(Bc)); + } +} + +// Every tail width of a deep block, which the ZA transposer packs: a ColMajor RHS tail and, through a RowMajor +// product, a transposed LHS tail. +template <typename Scalar> +static void test_deep_tail_panels() { + const Index MR = sme_mr<Scalar>(), NR = sme_nr<Scalar>(); + for (Index depth : {4 * NR, 6 * NR + 3}) { + for (Index tail = 1; tail < NR; ++tail) { + for (Index n : {tail, NR + tail}) { + const SmeColMajorMat<Scalar> A = SmeColMajorMat<Scalar>::Random(MR + 3, depth); + const SmeColMajorMat<Scalar> B = SmeColMajorMat<Scalar>::Random(depth, n); + SmeColMajorMat<Scalar> C = SmeColMajorMat<Scalar>::Random(MR + 3, n); + const SmeColMajorMat<Scalar> C0 = C; + C.noalias() += A * B; + VERIFY_IS_APPROX(C, C0 + A.lazyProduct(B)); + const SmeRowMajorMat<Scalar> Ar = SmeRowMajorMat<Scalar>::Random(n, depth); + const SmeRowMajorMat<Scalar> Br = SmeRowMajorMat<Scalar>::Random(depth, MR + 3); + SmeRowMajorMat<Scalar> Cr = SmeRowMajorMat<Scalar>::Random(n, MR + 3); + const SmeRowMajorMat<Scalar> Cr0 = Cr; + Cr.noalias() += Ar * Br; + VERIFY_IS_APPROX(Cr, Cr0 + Ar.lazyProduct(Br)); + } + } + } +} + // --------------------------------------------------------------------------- // Raw packed-buffer tests. // @@ -694,6 +1061,13 @@ sweep_pack_direct<Scalar, RowMajor>(n, d); } } + // Four-panel LHS copies with leftover panels and tails, over several transposing depth chunks. + for (int d : {35, 67, 97}) { + for (int n : {4 * MR, 5 * MR + 1, 8 * MR + 3}) { + sweep_pack_direct<Scalar, ColMajor>(n, d); + sweep_pack_direct<Scalar, RowMajor>(n, d); + } + } } template <typename Scalar> @@ -727,8 +1101,8 @@ const int T = sme_tile<Scalar>(); const int MR = sme_mr<Scalar>(); const int NR = sme_nr<Scalar>(); - std::vector<int> sizes = {1, T - 1, T, T + 1, MR - 1, MR, MR + 1, NR - 1, - NR, NR + 1, 2 * MR - 1, 2 * MR, 2 * MR + 1, 2 * NR - 1, 2 * NR, 2 * NR + 1}; + std::vector<int> sizes = {T / 2, T + T / 2, 1, T - 1, T, T + 1, MR - 1, MR, MR + 1, + NR - 1, NR, NR + 1, 2 * MR - 1, 2 * MR, 2 * MR + 1, 2 * NR - 1, 2 * NR, 2 * NR + 1}; std::sort(sizes.begin(), sizes.end()); sizes.erase(std::unique(sizes.begin(), sizes.end()), sizes.end()); return sizes; @@ -850,14 +1224,82 @@ } } +// Folding C into the complex alpha chain, (c_re + re*ar) - im*ai, overflows +// for acc = (h,h), alpha = (1,1), C = (3h,0) although the exact result +// (3h,2h) is finite (#3139 review). Widths straddle the one-vector slice +// width, which selects the store form; a real alpha takes the fused store. +// RowMajor GEMM transposes the computation, so the packed kernel is also +// called directly to reach the horizontal slice store. +template <typename Real, int Order> +static void test_complex_alpha_range() { + using Scalar = std::complex<Real>; + using Mat = Matrix<Scalar, Dynamic, Dynamic, Order>; + const int svl = internal::sme_packet_traits<Real>::size(); + const Real h = std::ldexp(Real(1), std::numeric_limits<Real>::max_exponent - 2); + const Index depth = 65; + const Scalar initial(3 * h, 0); + const Scalar alphas[] = {Scalar(1, 1), Scalar(-1, 0)}; + const Scalar expected[] = {Scalar(3 * h, 2 * h), Scalar(2 * h, -h)}; + for (int n : {svl / 2 - 1, svl / 2, svl / 2 + 1, svl, svl + 1}) { + if (n < 2) continue; + Mat a = Mat::Zero(n, depth), b = Mat::Zero(depth, n); + a.col(0).setConstant(Scalar(h, h)); + b.row(0).setOnes(); + const Mat ac = a.conjugate(), bc = b.conjugate(); + for (int t = 0; t < 2; ++t) { + const Scalar alpha = alphas[t]; + for (int conjugation = 0; conjugation < 4; ++conjugation) { + Mat c = Mat::Constant(n, n, initial); + if (conjugation == 0) c.noalias() += alpha * (a * b); + if (conjugation == 1) c.noalias() += alpha * (ac.conjugate() * b); + if (conjugation == 2) c.noalias() += alpha * (a * bc.conjugate()); + if (conjugation == 3) c.noalias() += alpha * (ac.conjugate() * bc.conjugate()); + for (Index j = 0; j < n; ++j) { + for (Index i = 0; i < n; ++i) { + VERIFY((std::isfinite)(c(i, j).real()) && (std::isfinite)(c(i, j).imag())); + VERIFY(c(i, j) == expected[t]); + } + } + } + // One packed panel; a larger runtime SVL may exceed its static width. + if (n <= numext::mini(sme_mr<Scalar>(), sme_nr<Scalar>())) { + std::vector<Scalar> pa(n * depth), pb(n * depth); + for (Index k = 0; k < depth; ++k) { + for (Index i = 0; i < n; ++i) { + set_packed(pa.data(), Index(n), k, i, a(i, k)); + set_packed(pb.data(), Index(n), k, i, b(k, i)); + } + } + Mat c = Mat::Constant(n, n, initial); + internal::sme_gebp_dispatch<Scalar, false, false>::run( + c.data(), Index(Order == RowMajor ? n : 1), Index(Order == RowMajor ? 1 : n), pa.data(), pb.data(), + Index(n), depth, Index(n), alpha, depth, depth, Index(0), Index(0)); + for (Index j = 0; j < n; ++j) { + for (Index i = 0; i < n; ++i) { + VERIFY((std::isfinite)(c(i, j).real()) && (std::isfinite)(c(i, j).imag())); + VERIFY(c(i, j) == expected[t]); + } + } + } + } + } +} + EIGEN_DECLARE_TEST(product_sme) { CALL_SUBTEST_1(test_whilelt_operand_types<float>()); + CALL_SUBTEST_1(test_packet_ops<float>()); CALL_SUBTEST_1(test_products<float>()); CALL_SUBTEST_1(test_conjugated_products<float>()); CALL_SUBTEST_1(test_symm_pack<float>()); CALL_SUBTEST_1(test_pack_direct<float>()); CALL_SUBTEST_1(test_mapper_fallback<float>()); CALL_SUBTEST_1(test_neon_small_blocks<float>()); + CALL_SUBTEST_1(test_direct_lhs<float>()); + CALL_SUBTEST_1(test_small_k_and_single_panel_rhs<float>()); + CALL_SUBTEST_1(test_deep_tail_panels<float>()); + CALL_SUBTEST_1(test_tiny_results<float>()); + CALL_SUBTEST_1(test_fp_flags<float>()); + CALL_SUBTEST_1(test_disjoint_parts<float>()); // double reaches the SME kernel and packers only with FEAT_SME_F64F64; the // product sweep is meaningful either way, but the packed-layout tests name @@ -866,10 +1308,17 @@ CALL_SUBTEST_2(test_conjugated_products<double>()); #ifdef EIGEN_VECTORIZE_SME_F64F64 CALL_SUBTEST_2(test_whilelt_operand_types<double>()); + CALL_SUBTEST_2(test_packet_ops<double>()); CALL_SUBTEST_2(test_symm_pack<double>()); CALL_SUBTEST_2(test_pack_direct<double>()); CALL_SUBTEST_2(test_mapper_fallback<double>()); CALL_SUBTEST_2(test_neon_small_blocks<double>()); + CALL_SUBTEST_2(test_direct_lhs<double>()); + CALL_SUBTEST_2(test_small_k_and_single_panel_rhs<double>()); + CALL_SUBTEST_2(test_deep_tail_panels<double>()); + CALL_SUBTEST_2(test_tiny_results<double>()); + CALL_SUBTEST_2(test_fp_flags<double>()); + CALL_SUBTEST_2(test_disjoint_parts<double>()); #endif CALL_SUBTEST_3(test_products<std::complex<float>>()); @@ -878,6 +1327,10 @@ CALL_SUBTEST_3(test_pack_direct<std::complex<float>>()); CALL_SUBTEST_3(test_mapper_fallback<std::complex<float>>()); CALL_SUBTEST_3(test_neon_small_blocks<std::complex<float>>()); + CALL_SUBTEST_3(test_fp_flags<std::complex<float>>()); + CALL_SUBTEST_3(test_disjoint_parts<std::complex<float>>()); + CALL_SUBTEST_3((test_complex_alpha_range<float, ColMajor>())); + CALL_SUBTEST_3((test_complex_alpha_range<float, RowMajor>())); // complex<double> accumulates into ZA.D tiles, so it needs FEAT_SME_F64F64 // exactly as double does. @@ -888,6 +1341,10 @@ CALL_SUBTEST_4(test_pack_direct<std::complex<double>>()); CALL_SUBTEST_4(test_mapper_fallback<std::complex<double>>()); CALL_SUBTEST_4(test_neon_small_blocks<std::complex<double>>()); + CALL_SUBTEST_4(test_fp_flags<std::complex<double>>()); + CALL_SUBTEST_4(test_disjoint_parts<std::complex<double>>()); + CALL_SUBTEST_4((test_complex_alpha_range<double, ColMajor>())); + CALL_SUBTEST_4((test_complex_alpha_range<double, RowMajor>())); #endif // A scalar type SME does not specialize, proving it still routes through the
diff --git a/test/product_threaded.cpp b/test/product_threaded.cpp index 6b5c960..6cb91d2 100644 --- a/test/product_threaded.cpp +++ b/test/product_threaded.cpp
@@ -31,6 +31,7 @@ int saved_threads_; }; +template <typename = void> void test_parallelize_gemm() { constexpr int n = 1024; constexpr int num_threads = 4; @@ -52,6 +53,7 @@ VERIFY_IS_APPROX(c, c_threaded); } +template <typename = void> void test_parallelize_gemm_varied() { constexpr int num_threads = 4; static ThreadPool pool(num_threads); @@ -102,6 +104,7 @@ } } +template <typename = void> void test_balanced_gemm_range() { static const Index totals[] = {0, 1, 3, 4, 7, 12, 63, 100, 4096, 8192, 9216}; static const Index part_counts[] = {1, 2, 3, 7, 8, 32, 64, 72}; @@ -144,6 +147,7 @@ VERIFY_IS_APPROX(c_serial, c_threaded); } +template <typename = void> void test_parallelize_gemm_indivisible() { // Shapes deliberately not divisible by the thread count, where the split has to spread the // remainder rather than append it to the last thread. @@ -154,9 +158,193 @@ verify_threaded_product<MatrixXf>(pool, 4099, 331, 64); } +// A scalar whose constructors set a magic value that operator= checks: the shared blocking buffers of a threaded +// product must be constructed before the packers assign into them (NumTraits::RequireInitialization). +struct TrackedScalar { + static constexpr unsigned kMagic = 0x5EEDu; + static int unconstructed_assignments; + double v; + unsigned magic; + TrackedScalar() : v(0), magic(kMagic) {} + TrackedScalar(double d) : v(d), magic(kMagic) {} + TrackedScalar(const TrackedScalar& o) : v(o.v), magic(kMagic) {} + TrackedScalar& operator=(const TrackedScalar& o) { + if (magic != kMagic) ++unconstructed_assignments; + v = o.v; + return *this; + } + TrackedScalar operator+(const TrackedScalar& o) const { return TrackedScalar(v + o.v); } + TrackedScalar operator-(const TrackedScalar& o) const { return TrackedScalar(v - o.v); } + TrackedScalar operator*(const TrackedScalar& o) const { return TrackedScalar(v * o.v); } + TrackedScalar operator-() const { return TrackedScalar(-v); } + TrackedScalar& operator+=(const TrackedScalar& o) { return *this = *this + o; } + TrackedScalar& operator*=(const TrackedScalar& o) { return *this = *this * o; } + bool operator==(const TrackedScalar& o) const { return v == o.v; } + bool operator!=(const TrackedScalar& o) const { return v != o.v; } + bool operator<(const TrackedScalar& o) const { return v < o.v; } +}; +int TrackedScalar::unconstructed_assignments = 0; + +namespace Eigen { +template <> +struct NumTraits<TrackedScalar> : GenericNumTraits<TrackedScalar> { + using Real = TrackedScalar; + using NonInteger = TrackedScalar; + using Nested = TrackedScalar; + static constexpr int IsComplex = 0; + static constexpr int IsInteger = 0; + static constexpr int IsSigned = 1; + static constexpr int RequireInitialization = 1; + static constexpr int ReadCost = 1; + static constexpr int AddCost = 1; + static constexpr int MulCost = 1; +}; +} // namespace Eigen + +template <typename = void> +void test_parallelize_gemm_require_initialization() { + constexpr int n = 160; + static ThreadPool pool(4); + Eigen::setGemmThreadPool(&pool); + using Mat = Matrix<TrackedScalar, Dynamic, Dynamic>; + const MatrixXd ad = MatrixXd::Random(n, n), bd = MatrixXd::Random(n, n); + const Mat a = ad.cast<TrackedScalar>(), b = bd.cast<TrackedScalar>(); + Mat c(n, n); + TrackedScalar::unconstructed_assignments = 0; + c.noalias() = a * b; + VERIFY_IS_EQUAL(TrackedScalar::unconstructed_assignments, 0); + const MatrixXd cd = c.unaryExpr([](const TrackedScalar& x) { return x.v; }); + VERIFY_IS_APPROX(cd, ad * bd); +} + +// Tiny results and thin column ranges on a pool: tiny results run on one thread, and every thread of a split product +// takes the same path. +template <typename Scalar> +void test_parallelize_gemm_tiny() { + static ThreadPool pool(4); + Eigen::setGemmThreadPool(&pool); + Eigen::setNbThreads(4); + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + for (Index m : {2, 4, 8}) + for (Index n : {3, 8, 17, 36}) + for (Index k : {16, 2000, 4096}) { + const Mat a = Mat::Random(m, k), b = Mat::Random(k, n); + Mat c(m, n); + c.noalias() = a * b; + VERIFY_IS_APPROX(c, a.lazyProduct(b)); + } + Eigen::setNbThreads(0); +} + +// Shapes the SME backend splits into disjoint parts per unit: odd sizes, a row-major result, and thin results split +// along their long side; compared with a serial product. +template <typename = void> +void test_parallelize_gemm_parts() { + static ThreadPool pool(4); + Eigen::setGemmThreadPool(&pool); + const int shapes[][3] = {{333, 517, 129}, {1024, 24, 512}, {24, 1024, 512}, {517, 333, 257}}; + for (const auto& s : shapes) { + const MatrixXf a = MatrixXf::Random(s[0], s[2]), b = MatrixXf::Random(s[2], s[1]); + MatrixXf c_serial(s[0], s[1]), c_threaded(s[0], s[1]); + { + ScopedSerialGemm serial; + c_serial.noalias() = a * b; + } + c_threaded.noalias() = a * b; + VERIFY_IS_APPROX(c_serial, c_threaded); + using RowMat = Matrix<float, Dynamic, Dynamic, RowMajor>; + const RowMat ar = a, br = b; + RowMat r_serial(s[0], s[1]), r_threaded(s[0], s[1]); + { + ScopedSerialGemm serial; + r_serial.noalias() = ar * br; + } + r_threaded.noalias() = ar * br; + VERIFY_IS_APPROX(r_serial, r_threaded); + VERIFY_IS_APPROX(r_threaded, RowMat(c_serial)); + } +} + +// The disjoint parts for every scalar the SME kernel serves, including shapes whose split side has fewer chunks than +// threads, and a tall result that the 8-panel rule splits by columns instead of rows. +template <typename Scalar> +void test_parallelize_gemm_parts_types() { + static ThreadPool pool(4); + Eigen::setGemmThreadPool(&pool); + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + using RowMat = Matrix<Scalar, Dynamic, Dynamic, RowMajor>; + const int shapes[][3] = {{301, 259, 131}, {1024, 20, 384}, {20, 1024, 384}, {40, 700, 300}, + {48, 32, 400}, {300, 40, 2048}, {200, 40, 4096}}; + for (const auto& s : shapes) { + const Mat a = Mat::Random(s[0], s[2]), b = Mat::Random(s[2], s[1]); + Mat c_serial(s[0], s[1]), c_threaded(s[0], s[1]); + { + ScopedSerialGemm serial; + c_serial.noalias() = a * b.conjugate(); + } + c_threaded.noalias() = a * b.conjugate(); + VERIFY_IS_APPROX(c_serial, c_threaded); + RowMat r_threaded(s[0], s[1]); + r_threaded.noalias() = RowMat(a) * b.conjugate(); + VERIFY_IS_APPROX(Mat(r_threaded), c_serial); + } +} + +// A block destination accumulated with a scale factor, and a fixed-size result, whose preallocated buffers keep the +// shared session. +template <typename = void> +void test_parallelize_gemm_parts_blocks() { + static ThreadPool pool(4); + Eigen::setGemmThreadPool(&pool); + const MatrixXf a = MatrixXf::Random(517, 301), b = MatrixXf::Random(301, 283); + MatrixXf big = MatrixXf::Random(530, 300); + MatrixXf ref = big; + { + ScopedSerialGemm serial; + ref.block(5, 7, 517, 283).noalias() += 1.5f * a * b; + } + big.block(5, 7, 517, 283).noalias() += 1.5f * a * b; + VERIFY_IS_APPROX(big, ref); + using Fixed = Matrix<float, 160, 160>; + const Fixed fa = Fixed::Random(), fb = Fixed::Random(); + Fixed fc, fr; + { + ScopedSerialGemm serial; + fr.noalias() = fa * fb; + } + fc.noalias() = fa * fb; + VERIFY_IS_APPROX(fc, fr); +} + +// Runs a test with the given SME unit count: a nonzero count splits products on the SME kernel into disjoint parts, +// 0 keeps the shared parallel session, so both paths are covered on every host. +template <typename Test> +void with_sme_units(int units, Test test) { + const int saved = Eigen::nbSmeUnits(); + Eigen::setNbSmeUnits(units); + test(); + Eigen::setNbSmeUnits(saved); +} + EIGEN_DECLARE_TEST(product_threaded) { - CALL_SUBTEST_1(test_parallelize_gemm()); - CALL_SUBTEST_2(test_parallelize_gemm_varied()); - CALL_SUBTEST_3(test_balanced_gemm_range()); - CALL_SUBTEST_4(test_parallelize_gemm_indivisible()); + // Off SME the unit count does nothing, so only the default runs there. +#ifdef EIGEN_VECTORIZE_SME + for (int units : {2, 3, 0}) { +#else + for (int units : {0}) { +#endif + EIGEN_UNUSED_VARIABLE(units); + CALL_SUBTEST_6(with_sme_units(units, test_parallelize_gemm_tiny<float>)); + CALL_SUBTEST_6(with_sme_units(units, test_parallelize_gemm_tiny<double>)); + CALL_SUBTEST_1(with_sme_units(units, test_parallelize_gemm<>)); + CALL_SUBTEST_2(with_sme_units(units, test_parallelize_gemm_varied<>)); + CALL_SUBTEST_4(with_sme_units(units, test_parallelize_gemm_indivisible<>)); + CALL_SUBTEST_7(with_sme_units(units, test_parallelize_gemm_parts<>)); + CALL_SUBTEST_8(with_sme_units(units, test_parallelize_gemm_parts_types<double>)); + CALL_SUBTEST_8(with_sme_units(units, test_parallelize_gemm_parts_types<std::complex<float>>)); + CALL_SUBTEST_8(with_sme_units(units, test_parallelize_gemm_parts_types<std::complex<double>>)); + CALL_SUBTEST_9(with_sme_units(units, test_parallelize_gemm_parts_blocks<>)); + } + CALL_SUBTEST_3(test_balanced_gemm_range<>()); + CALL_SUBTEST_5(test_parallelize_gemm_require_initialization<>()); }
diff --git a/test/product_threshold.cpp b/test/product_threshold.cpp new file mode 100644 index 0000000..ce00399 --- /dev/null +++ b/test/product_threshold.cpp
@@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +// Defining only the run-time threshold must leave the compile-time fixed-size one at its default (#3119). 30 lies +// between the two defaults, so a bound derived from it as 2 * 30 would move 14 x 14 x 14 to the coeff-based path. +#define EIGEN_GEMM_TO_COEFFBASED_THRESHOLD 30 +#include "main.h" + +static_assert(EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD == 40, + "the fixed-size default must not follow the runtime one"); + +template <typename Scalar> +void fixed_size_threshold_is_independent() { +#ifdef EIGEN_VECTORIZE_SME + constexpr int kThreshold = internal::sme_has_gebp_kernel<Scalar, Scalar>::value + ? EIGEN_SME_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD + : EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD; +#else + constexpr int kThreshold = EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD; +#endif + constexpr int kGemmSide = (kThreshold + 2) / 3; + using Below = Matrix<Scalar, kGemmSide - 1, kGemmSide - 1>; + using At = Matrix<Scalar, kGemmSide, kGemmSide>; + STATIC_CHECK((internal::product_type<At, At>::FixedSizeThreshold == kThreshold)); + STATIC_CHECK((internal::product_type<Below, Below>::value == CoeffBasedProductMode)); + STATIC_CHECK((internal::product_type<At, At>::value == GemmProduct)); + + const Below a = Below::Random(), b = Below::Random(); + Below c; + c.noalias() = a * b; + VERIFY_IS_APPROX(c, a.lazyProduct(b)); + // 3 * kGemmSide >= 30: the run-time bound keeps it on the GEMM path. + const At d = At::Random(), e = At::Random(); + At f; + f.noalias() = d * e; + VERIFY_IS_APPROX(f, d.lazyProduct(e)); +} + +EIGEN_DECLARE_TEST(product_threshold) { + CALL_SUBTEST(fixed_size_threshold_is_independent<float>()); + CALL_SUBTEST(fixed_size_threshold_is_independent<double>()); + CALL_SUBTEST(fixed_size_threshold_is_independent<std::complex<float>>()); + CALL_SUBTEST(fixed_size_threshold_is_independent<std::complex<double>>()); +}
diff --git a/test/realview.cpp b/test/realview.cpp index 159c3fe..ee5b5a9 100644 --- a/test/realview.cpp +++ b/test/realview.cpp
@@ -202,6 +202,47 @@ test_edge_cases_impl<ComplexScalar>::run(); } +// The parts of non-adjacent complex coefficients have no uniform stride, so their RealView has no direct access. +template <typename ComplexScalar> +void test_strided_realview(const ComplexScalar&) { + using RealScalar = typename NumTraits<ComplexScalar>::Real; + using ComplexMatrix = Matrix<ComplexScalar, Dynamic, Dynamic>; + using RealRowVector = Matrix<RealScalar, 1, Dynamic>; + const ComplexMatrix a = ComplexMatrix::Random(6, 8); + VERIFY((int(internal::traits<std::decay_t<decltype(a.row(2).realView())>>::Flags) & DirectAccessBit) == 0); + VERIFY((int(internal::traits<std::decay_t<decltype(a.col(2).realView())>>::Flags) & DirectAccessBit) != 0); + + RealRowVector expected(2 * a.cols()); + for (Index j = 0; j < a.cols(); ++j) { + expected(2 * j) = numext::real(a(2, j)); + expected(2 * j + 1) = numext::imag(a(2, j)); + } + const Ref<const RealRowVector, 0, InnerStride<>> row = a.row(2).realView(); + VERIFY_IS_CWISE_EQUAL(row, expected); + + // A writable view keeps LvalueBit, so its coeff() and wrapping evaluators return references into the complex + // storage. + ComplexMatrix b = a; + auto bRow = b.row(2); + auto bView = bRow.realView(); + const auto& constView = bView; + for (Index j = 0; j < constView.size(); ++j) VERIFY_IS_EQUAL(constView.coeff(j), expected(j)); + VERIFY_IS_CWISE_EQUAL(RealRowVector(b.row(2).realView().array().matrix()), expected); + VERIFY_IS_EQUAL(b.row(2).realView().segment(1, 5).sum(), expected.segment(1, 5).sum()); + + // Products read a direct-access vector operand through data() and innerStride(). + using RealMatrix = Matrix<RealScalar, Dynamic, Dynamic>; + const RealMatrix e = RealMatrix::Random(3, 2 * a.cols()); + VERIFY_IS_APPROX(e * a.row(2).realView().transpose(), e * expected.transpose()); + + // data() keeps the constness of the viewed storage, including through an IndexedView of a const matrix. + const RealScalar* aCol1 = reinterpret_cast<const RealScalar*>(a.data() + a.rows()); + VERIFY_IS_EQUAL(a(placeholders::all, seq(1, 7, 3)).realView().data(), aCol1); + auto aCols = a(placeholders::all, seq(1, 7, 3)); + const auto aColsView = aCols.realView(); + VERIFY_IS_EQUAL(aColsView.data(), aCol1); +} + template <typename Scalar, int Rows, int Cols, int MaxRows = Rows, int MaxCols = Cols> void test_realview_readonly() { // if Rows == 1, don't test ColMajor as it is not a valid array @@ -262,7 +303,9 @@ CALL_SUBTEST_9((test_realview<1, Dynamic>())); CALL_SUBTEST_10((test_realview<1, 1>())); CALL_SUBTEST_11(test_edge_cases(std::complex<float>())); + CALL_SUBTEST_11(test_strided_realview(std::complex<float>())); CALL_SUBTEST_12(test_edge_cases(std::complex<double>())); + CALL_SUBTEST_12(test_strided_realview(std::complex<double>())); CALL_SUBTEST_13((test_realview_long_double<Dynamic, Dynamic, Dynamic, Dynamic>())); CALL_SUBTEST_13((test_realview_long_double<17, 19, 17, 19>())); }
diff --git a/test/redux_bounded.cpp b/test/redux_bounded.cpp index 8ab7ad8..c446702 100644 --- a/test/redux_bounded.cpp +++ b/test/redux_bounded.cpp
@@ -77,6 +77,19 @@ VERIFY_IS_EQUAL((a + b).redux(BoundedFirst()), 5); VERIFY_IS_EQUAL((b + a).redux(BoundedFirst()), 5); + const auto wrapped_array = (a + b).array(); + const auto wrapped_matrix = (a.array() + b.array()).matrix(); + using ArrayBounds = internal::redux_max_size<internal::remove_all_t<decltype(wrapped_array)>>; + using MatrixBounds = internal::redux_max_size<internal::remove_all_t<decltype(wrapped_matrix)>>; + STATIC_CHECK(ArrayBounds::Rows == 4 && ArrayBounds::Cols == 4 && ArrayBounds::Size == 16); + STATIC_CHECK(MatrixBounds::Rows == 4 && MatrixBounds::Cols == 4 && MatrixBounds::Size == 16); + STATIC_CHECK(decltype(wrapped_array)::MaxSizeAtCompileTime == Dynamic); + STATIC_CHECK(decltype(wrapped_matrix)::MaxSizeAtCompileTime == Dynamic); + VERIFY_IS_EQUAL(wrapped_array.sum(), 60); + VERIFY_IS_EQUAL(wrapped_matrix.sum(), 60); + VERIFY_IS_EQUAL(wrapped_array.redux(BoundedFirst()), 5); + VERIFY_IS_EQUAL(wrapped_matrix.redux(BoundedFirst()), 5); + const Matrix<bool, Dynamic, Dynamic, Order> condition = Matrix<bool, Dynamic, Dynamic, Order>::Constant(3, 4, true); const auto selected = condition.select(a, b); const auto reversed = condition.select(b, a); @@ -129,6 +142,31 @@ STATIC_CHECK(internal::redux_max_size<internal::remove_all_t<decltype(right)>>::Size == Capacity); VERIFY_IS_EQUAL(left.sum(), expected); VERIFY_IS_EQUAL(right.sum(), expected); + const auto squared = left.cwiseAbs2(); + const auto negated = -squared; + STATIC_CHECK(decltype(squared)::MaxSizeAtCompileTime == Dynamic); + STATIC_CHECK(internal::redux_max_size<internal::remove_all_t<decltype(squared)>>::Size == Capacity); + STATIC_CHECK(internal::redux_max_size<internal::remove_all_t<decltype(negated)>>::Size == Capacity); + Scalar expectedSquared = 0; + for (Index i = 0; i < size; ++i) { + const Scalar value = bounded(i) * dynamic(i); + expectedSquared += value * value; + } + VERIFY_IS_EQUAL(squared.sum(), expectedSquared); + VERIFY_IS_EQUAL(negated.sum(), -expectedSquared); + VERIFY_IS_EQUAL(left.squaredNorm(), expectedSquared); + + const auto wrapped = left.array().square(); + auto array_product = dynamic.array() * bounded.array(); + const auto matrix = array_product.matrix(); + const auto nested = matrix.array().square().matrix(); + STATIC_CHECK(internal::redux_max_size<internal::remove_all_t<decltype(wrapped)>>::Size == Capacity); + STATIC_CHECK(internal::redux_max_size<internal::remove_all_t<decltype(matrix)>>::Size == Capacity); + STATIC_CHECK(internal::redux_max_size<internal::remove_all_t<decltype(nested)>>::Size == Capacity); + VERIFY_IS_EQUAL(wrapped.sum(), expectedSquared); + VERIFY_IS_EQUAL(matrix.sum(), expected); + VERIFY_IS_EQUAL(matrix.squaredNorm(), expectedSquared); + VERIFY_IS_EQUAL(nested.sum(), expectedSquared); } }
diff --git a/test/redux_bounded_compile.cpp b/test/redux_bounded_compile.cpp index 98d45c1..b038609 100644 --- a/test/redux_bounded_compile.cpp +++ b/test/redux_bounded_compile.cpp
@@ -2,7 +2,10 @@ // SPDX-License-Identifier: MPL-2.0 #include <Eigen/Core> +#include <functional> +// Scalar only: InnerProduct's packet-enabled unit-stride remap loses this product's bound. +#ifdef EIGEN_DONT_VECTORIZE using BoundedVector4d = Eigen::Matrix<double, Eigen::Dynamic, 1, Eigen::ColMajor, 4, 1>; using BoundedMatrix4d = Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor, 4, 4>; @@ -15,5 +18,40 @@ } return transformed; } +#endif -int main() {} +using BoundedVector3d = Eigen::Matrix<double, Eigen::Dynamic, 1, Eigen::ColMajor, 3, 1>; + +std::function<Eigen::VectorXd(Eigen::Index)> evaluate_position; + +template <int Mode> +EIGEN_ALWAYS_INLINE double bounded_residual(const BoundedVector3d& target, Eigen::Index size) { + const Eigen::VectorXd position = evaluate_position(size); + EIGEN_IF_CONSTEXPR (Mode == 0) { + return (position - target.head(size)).norm(); + } else EIGEN_IF_CONSTEXPR (Mode == 1) { + return Eigen::numext::sqrt((position - target.head(size)).array().square().sum()); + } else { + return (position.array() - target.head(size).array()).matrix().norm(); + } +} + +template <int Mode> +Eigen::VectorXd bounded_residuals(const Eigen::MatrixXd& points, Eigen::Index size) { + Eigen::VectorXd result(points.cols()); + for (Eigen::Index i = 0; i < points.cols(); ++i) + result(i) = bounded_residual<Mode>(BoundedVector3d(points.col(i)), size); + return result; +} + +int main() { + evaluate_position = [](Eigen::Index size) -> Eigen::VectorXd { return Eigen::VectorXd::Zero(size); }; + for (Eigen::Index size = 1; size <= 3; ++size) { + const Eigen::MatrixXd points = Eigen::MatrixXd::Ones(size, 2); + for (const Eigen::VectorXd& result : + {bounded_residuals<0>(points, size), bounded_residuals<1>(points, size), bounded_residuals<2>(points, size)}) { + for (Eigen::Index i = 0; i < result.size(); ++i) + if (Eigen::numext::abs(result(i) - Eigen::numext::sqrt(double(size))) > 1e-12) return 1; + } + } +}
diff --git a/test/redux_bounded_compile_vectorized.cpp b/test/redux_bounded_compile_vectorized.cpp new file mode 100644 index 0000000..c9279d0 --- /dev/null +++ b/test/redux_bounded_compile_vectorized.cpp
@@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include "redux_bounded_compile.cpp" // NOLINT(bugprone-suspicious-include)
diff --git a/test/safe_scaling.cpp b/test/safe_scaling.cpp index 74eae29..3af70e4 100644 --- a/test/safe_scaling.cpp +++ b/test/safe_scaling.cpp
@@ -494,7 +494,196 @@ VERIFY(same_bits(Scaling::recover_flushed_max_coeff(Matrix<Scalar, 2, 2>::Zero(), RealScalar(0)), RealScalar(0))); } +// A strided view reduces coefficient by coefficient, and under Arm FZ a scalar max(a, b) is a compare and a select +// that keeps the first subnormal operand rather than producing a zero: the rescan must not trust a subnormal maximum. +// Every flush-to-zero mode, with the largest magnitude last. +template <typename Scalar> +void check_recover_flushed_scalar_max_coeff() { + using Binary = internal::binary_floating_point_traits<Scalar>; + using Bits = typename Binary::Bits; + using Scaling = internal::safe_scaling<Scalar>; + using VectorType = Matrix<Scalar, Dynamic, 1>; + VectorType storage = VectorType::Zero(6); + const Bits significands[3] = {Bits(5), Bits(8), Bits(23)}; + for (Index i = 0; i < 3; ++i) storage(2 * i) = numext::bit_cast<Scalar>(significands[i]); + const Map<const VectorType, 0, InnerStride<2>> strided(storage.data(), 3); + forEachFlushToZeroMode([&](FlushToZeroMode) { + const Scalar maxCoeff = strided.cwiseAbs().maxCoeff(); + VERIFY(same_bits(Scaling::recover_flushed_max_coeff(strided, maxCoeff), numext::bit_cast<Scalar>(Bits(23)))); + VERIFY_IS_EQUAL(internal::binary_frexp_exponent(Scaling::recover_flushed_max_coeff(strided, maxCoeff)), + 5 + std::numeric_limits<Scalar>::min_exponent - std::numeric_limits<Scalar>::digits); + }); +} + +// Each flush-to-zero mode the host selects does what its name says, read from the results of two probes through the +// representation, since a comparison under DAZ reads any subnormal as zero; the modes restore the ambient state. +template <typename Scalar> +void check_flush_to_zero_modes() { + using Binary = internal::binary_floating_point_traits<Scalar>; + const auto resultsFlushed = []() { return Binary::bits(underflowProbe<Scalar>()) == 0; }; + const auto inputsFlushed = []() { return Binary::bits(subnormalInputProbe<Scalar>()) == 0; }; + const bool resultsBefore = resultsFlushed(), inputsBefore = inputsFlushed(); + int modes = 0; + forEachFlushToZeroMode([&](FlushToZeroMode mode) { + ++modes; + VERIFY_IS_EQUAL(resultsFlushed(), flushToZeroModeIncludes(mode, FlushToZeroMode::Results)); + VERIFY_IS_EQUAL(inputsFlushed(), flushToZeroModeIncludes(mode, FlushToZeroMode::Inputs)); + VERIFY_IS_EQUAL(ScopedFlushToZero::hardwareFlushesSubnormalInputs(), + flushToZeroModeIncludes(mode, FlushToZeroMode::Inputs)); + }); + VERIFY(modes >= 1); + VERIFY_IS_EQUAL(resultsFlushed(), resultsBefore); + VERIFY_IS_EQUAL(inputsFlushed(), inputsBefore); + // The default request flushes at least results, as it always did. + { + const ScopedFlushToZero flushToZero; + if (flushToZero.isSupported()) { + VERIFY(flushToZeroModeIncludes(flushToZero.mode(), FlushToZeroMode::Results)); + VERIFY(resultsFlushed()); + } + } + VERIFY_IS_EQUAL(resultsFlushed(), resultsBefore); +} + +// value * 2^exponent through integer significands agrees bit for bit with the correctly rounded ldexp for every input +// class and every exponent that changes the result, and keeps doing so in every flush-to-zero mode, where the +// references are taken beforehand. Beyond the exponent range it saturates to the signed zero or infinity ldexp returns. +template <typename Scalar> +void check_scale_binary_by_exponent() { + using Binary = internal::binary_floating_point_traits<Scalar>; + using Bits = typename Binary::Bits; + using Scaling = internal::safe_scaling<Scalar>; + // 2^recovery is the min / eps threshold check_unscale_recovery_threshold() exercises. + VERIFY_IS_EQUAL(numext::ldexp(Scalar(1), Scaling::subnormal_recovery_exponent()), + (std::numeric_limits<Scalar>::min)() / NumTraits<Scalar>::epsilon()); + + // Subnormals down to denorm_min, the normal boundary, full fractions, and the largest finite value. + const Bits magnitudes[] = {Bits(1), + Bits(3), + Binary::kExponentUnit >> 1, + Binary::kFractionMask, + Binary::kExponentUnit, + Binary::kExponentUnit | Bits(1), + Binary::kExponentUnit + (Binary::kExponentUnit >> 1), + (Binary::kExponentUnit << 1) - Bits(1), + Binary::bits(Scalar(0.75)), + Binary::bits(Scalar(1)), + Binary::bits(Scalar(1.5)), + Binary::bits((std::numeric_limits<Scalar>::max)())}; + // Every exponent from full underflow to full overflow, then values that only the clamp can handle. + const int span = std::numeric_limits<Scalar>::max_exponent - std::numeric_limits<Scalar>::min_exponent + + std::numeric_limits<Scalar>::digits + 3; + std::vector<int> exponents; + for (int e = -span; e <= span; ++e) exponents.push_back(e); + for (int e : {1 << 24, 1 << 30, (std::numeric_limits<int>::max)()}) { + exponents.push_back(e); + exponents.push_back(-e); + } + exponents.push_back((std::numeric_limits<int>::min)()); + + struct Case { + Bits value; + int exponent; + Bits expected; + Bits expectedUnit; // ldexp(1, exponent) + int frexpExponent; // of value + Bits frexpSignificand; + }; + std::vector<Case> cases; + for (Bits magnitude : magnitudes) { + for (int e : exponents) { + for (Bits sign : {Bits(0), Bits(Binary::kSignBit)}) { + const Scalar value = numext::bit_cast<Scalar>(sign | magnitude); + int frexpExponent = 0; + EIGEN_USING_STD(frexp); + const Scalar frexpSignificand = frexp(value, &frexpExponent); + cases.push_back(Case{sign | magnitude, e, Binary::bits(numext::ldexp(value, e)), + Binary::bits(numext::ldexp(Scalar(1), e)), frexpExponent, Binary::bits(frexpSignificand)}); + } + } + } + for (const Scalar special : {Scalar(0), -Scalar(0), std::numeric_limits<Scalar>::infinity(), + -std::numeric_limits<Scalar>::infinity(), std::numeric_limits<Scalar>::quiet_NaN()}) { + for (int e : {-1000, -1, 0, 1, 1000}) { + cases.push_back(Case{Binary::bits(special), e, Binary::bits(special), Binary::bits(numext::ldexp(Scalar(1), e)), + 0, Binary::bits(special)}); + } + } + + const auto check = [&]() { + for (const Case& c : cases) { + const Scalar actual = internal::scale_binary_by_exponent(numext::bit_cast<Scalar>(c.value), c.exponent); + VERIFY_IS_EQUAL(Binary::bits(actual), c.expected); + const std::complex<Scalar> complexActual = internal::scale_binary_by_exponent( + std::complex<Scalar>(numext::bit_cast<Scalar>(c.value), Scalar(1)), c.exponent); + VERIFY_IS_EQUAL(Binary::bits(complexActual.real()), c.expected); + VERIFY_IS_EQUAL(Binary::bits(complexActual.imag()), c.expectedUnit); + // The representation-based classifiers agree with the IEEE ones on every value. + const Scalar value = numext::bit_cast<Scalar>(c.value); + VERIFY_IS_EQUAL(Binary::bits(internal::ldexp_preserving_subnormals(value, c.exponent)), c.expected); + const Bits magnitude = c.value & ~Binary::kSignBit; + VERIFY_IS_EQUAL(numext::is_exactly_zero_no_flush(value), magnitude == 0); + VERIFY_IS_EQUAL(internal::is_subnormal_magnitude(value), magnitude != 0 && magnitude < Binary::kExponentUnit); + if (magnitude == c.value && + (numext::isfinite)(value)) { // non-negative: the maximum against every larger power of two + VERIFY_IS_EQUAL(Binary::bits(internal::max_preserving_subnormals(value, numext::bit_cast<Scalar>(Bits(1)))), + magnitude == 0 ? Bits(1) : magnitude); + } + // frexp from the representation agrees with frexp on every finite value; zeros give exponent 0. + if ((numext::isfinite)(value)) { + int frexpExponent = 0; + const Scalar significand = internal::binary_frexp(value, frexpExponent); + VERIFY_IS_EQUAL(frexpExponent, c.frexpExponent); + VERIFY_IS_EQUAL(Binary::bits(significand), c.frexpSignificand); + VERIFY_IS_EQUAL(internal::binary_frexp_exponent(value), c.frexpExponent); + } + } + // A normal power-of-two factor is the exponent form of the same scaling. + for (int e = std::numeric_limits<Scalar>::min_exponent - 1; e < std::numeric_limits<Scalar>::max_exponent; ++e) { + const Scalar factor = numext::bit_cast<Scalar>(Bits(e + Binary::kExponentBias) << Binary::kFractionBits); + for (Bits magnitude : magnitudes) { + const Scalar value = numext::bit_cast<Scalar>(magnitude); + VERIFY_IS_EQUAL(Binary::bits(internal::scale_binary_by_power_of_two(value, factor)), + Binary::bits(internal::scale_binary_by_exponent(value, e))); + } + } + // The functor is the coefficient-wise scalar form, complex components included. + Matrix<std::complex<Scalar>, 3, 1> input; + input << std::complex<Scalar>(numext::bit_cast<Scalar>(Bits(3)), numext::bit_cast<Scalar>(Binary::kFractionMask)), + std::complex<Scalar>(Scalar(1.5), -Scalar(0.75)), + std::complex<Scalar>((std::numeric_limits<Scalar>::max)(), numext::bit_cast<Scalar>(Bits(1))); + for (int e : {-(std::numeric_limits<Scalar>::digits + 1), -3, 5, std::numeric_limits<Scalar>::max_exponent}) { + const Matrix<std::complex<Scalar>, 3, 1> scaled = input.unaryExpr(internal::scale_by_exponent_op<Scalar>(e)); + for (Index i = 0; i < input.size(); ++i) { + VERIFY(same_bits(scaled(i), internal::scale_binary_by_exponent(input(i), e))); + } + } + }; + forEachFlushToZeroMode([&](FlushToZeroMode) { check(); }); +} + +// The scalars without a representation path take the C library. +void check_preserving_subnormals_fallbacks() { + VERIFY_IS_EQUAL(internal::ldexp_preserving_subnormals(1.5L, 3), 12.0L); + VERIFY_IS_EQUAL(internal::frexp_exponent_preserving_subnormals(12.0L), 4); + int exponent = 0; + VERIFY_IS_EQUAL(internal::frexp_preserving_subnormals(-12.0L, exponent), -0.75L); + VERIFY_IS_EQUAL(exponent, 4); + VERIFY_IS_EQUAL(internal::frexp_exponent_preserving_subnormals(0.0L), 0); + VERIFY(numext::is_exactly_zero_no_flush(0.0L)); + VERIFY(!internal::is_subnormal_magnitude(1.0L)); + VERIFY(internal::is_subnormal_magnitude(std::numeric_limits<long double>::denorm_min())); + VERIFY_IS_EQUAL(internal::max_preserving_subnormals(1.5L, 2.5L), 2.5L); +} + EIGEN_DECLARE_TEST(safe_scaling) { + CALL_SUBTEST(check_flush_to_zero_modes<float>()); + CALL_SUBTEST(check_flush_to_zero_modes<double>()); + CALL_SUBTEST(check_recover_flushed_scalar_max_coeff<float>()); + CALL_SUBTEST(check_recover_flushed_scalar_max_coeff<double>()); + CALL_SUBTEST(check_scale_binary_by_exponent<float>()); + CALL_SUBTEST(check_scale_binary_by_exponent<double>()); + CALL_SUBTEST(check_preserving_subnormals_fallbacks()); CALL_SUBTEST(check_identity_scaling<float>()); CALL_SUBTEST(check_identity_scaling<double>()); CALL_SUBTEST(check_identity_scaling<std::complex<float>>());
diff --git a/test/scaled_permutation.cpp b/test/scaled_permutation.cpp index eab4ac0..e94dcae 100644 --- a/test/scaled_permutation.cpp +++ b/test/scaled_permutation.cpp
@@ -143,7 +143,11 @@ VERIFY_IS_EQUAL(MatrixType(a - scaled), MatrixType(a - scaledDense)); VERIFY_IS_EQUAL(MatrixType(scaled - a), MatrixType(scaledDense - a)); VERIFY_IS_APPROX((a + scaled) * x, (a + scaledDense) * x); - VERIFY_IS_APPROX((a + scaled).sum(), (a + scaledDense).sum()); + // The lazy sum reduces in scalar order and the dense one in packets, so the results differ by rounding of order + // eps * sum|t|, not of order eps * |result|, which cancellation can make arbitrarily small. Measured below 2 eps. + const RealScalar sumMagnitude = (a + scaledDense).cwiseAbs().sum(); + VERIFY(numext::abs((a + scaled).sum() - (a + scaledDense).sum()) <= + 8 * NumTraits<RealScalar>::epsilon() * sumMagnitude); // Storage and setters. ScaledType identity(size);
diff --git a/test/schur_real.cpp b/test/schur_real.cpp index 11d467f..6d0390e 100644 --- a/test/schur_real.cpp +++ b/test/schur_real.cpp
@@ -99,6 +99,39 @@ } } +// Structured inputs larger than one window of the Francis step's deferred left updates (32 reflectors). The +// permutation, companion and Jordan matrices give reflectors with exactly zero bulges (tau == 0), which the deferred +// path must skip as the immediate one does. +template <typename Scalar> +void real_schur_structured_windows() { + using MatrixType = Matrix<Scalar, Dynamic, Dynamic>; + for (Index n : {Index(40), Index(70)}) { + const MatrixType identity = MatrixType::Identity(n, n); + // Backward stable: ||A - U T U^T|| and ||U^T U - I|| are O(n eps) relative to ||A|| and 1. + const Scalar bound = Scalar(10 * n) * NumTraits<Scalar>::epsilon(); + MatrixType cyclic = MatrixType::Zero(n, n); + for (Index i = 0; i < n; ++i) cyclic(i, (i + 1) % n) = Scalar(1); + MatrixType companion = MatrixType::Zero(n, n); + companion.diagonal(-1).setOnes(); + companion.col(n - 1).setRandom(); + MatrixType jordan = MatrixType::Identity(n, n) * Scalar(2); + jordan.diagonal(1).setOnes(); + MatrixType decoupled = MatrixType::Random(n, n).template triangularView<Upper>(); + decoupled.diagonal(-1).setRandom(); + decoupled(n / 2, n / 2 - 1) = Scalar(0); + for (const MatrixType* a : {&cyclic, &companion, &jordan, &decoupled}) { + RealSchur<MatrixType> rs(*a); + VERIFY_IS_EQUAL(rs.info(), Success); + const MatrixType& T = rs.matrixT(); + const MatrixType& U = rs.matrixU(); + verifyIsQuasiTriangular(T); + VERIFY((*a - U * T * U.transpose()).norm() <= bound * a->norm()); + VERIFY((U.transpose() * U - identity).norm() <= bound); + } + } +} + +template <typename = void> void test_bug2633() { Eigen::MatrixXd A(4, 4); A << 0, 0, 0, -2, 1, 0, 0, -0, 0, 1, 0, 2, 0, 0, 2, -0; @@ -106,6 +139,7 @@ VERIFY(schur.info() == Eigen::Success); } +template <typename = void> void real_schur_power_of_two_scaling() { // Reciprocal scaling rounds the smaller diagonal entry up by one ULP. Matrix2f matrix = Matrix2f::Zero(); @@ -275,8 +309,8 @@ // Test problem size constructors CALL_SUBTEST_5(RealSchur<MatrixXf>(10)); - CALL_SUBTEST_6((test_bug2633())); - CALL_SUBTEST_6((real_schur_power_of_two_scaling())); + CALL_SUBTEST_6((test_bug2633<>())); + CALL_SUBTEST_6((real_schur_power_of_two_scaling<>())); CALL_SUBTEST_6(real_schur_subnormal_restoration<Matrix2f>()); CALL_SUBTEST_6(real_schur_subnormal_restoration<Matrix2d>()); CALL_SUBTEST_7((schur_workspace_stride<float, ColMajor>())); @@ -287,4 +321,6 @@ CALL_SUBTEST_12((schur_workspace_ref<double, ColMajor>())); CALL_SUBTEST_13((schur_workspace_ref<float, RowMajor>())); CALL_SUBTEST_14((schur_workspace_ref<double, RowMajor>())); + CALL_SUBTEST_15((real_schur_structured_windows<float>())); + CALL_SUBTEST_15((real_schur_structured_windows<double>())); }
diff --git a/test/sparse_product.cpp b/test/sparse_product.cpp index 8cdc09d..c819368 100644 --- a/test/sparse_product.cpp +++ b/test/sparse_product.cpp
@@ -10,13 +10,14 @@ static long int nb_temporaries; +template <typename = void> inline void on_temporary_creation() { // here's a great place to set a breakpoint when debugging failures in this test! nb_temporaries++; } #define EIGEN_SPARSE_CREATE_TEMPORARY_PLUGIN \ - { on_temporary_creation(); } + { on_temporary_creation<>(); } #include "sparse.h" @@ -590,6 +591,7 @@ } } +template <typename = void> void test_mixed_storage() { test_mixed_storage_imp<RowMajor, RowMajor, RowMajor>(); test_mixed_storage_imp<RowMajor, RowMajor, ColMajor>(); @@ -606,6 +608,7 @@ // (index, value) nodes depending on the estimated density. Both storage modes // must construct, move, and destroy the coefficients properly for // non-trivially-copyable scalars (issue #2873). +template <typename = void> void test_pruned_product_custom_scalar() { typedef SparseMatrix<AnnoyingScalar> SpMat; typedef Matrix<AnnoyingScalar, Dynamic, Dynamic> DenseMat; @@ -648,6 +651,7 @@ // reaches it through init(), which drops the list too. Neither may abandon the // coefficients the dropped nodes own, and neither may leave a node behind for // the next coeffRef() to construct over. +template <typename = void> void test_ambivector_discard_custom_scalar() { typedef internal::AmbiVector<AnnoyingScalar, int> AmbiVec; const Index n = 32; @@ -678,6 +682,7 @@ // resize() reuses its allocation whenever the new size still fits, and the // sub-vector bounds an iterator walks describe the size the vector had before. +template <typename = void> void test_ambivector_resize_bounds() { typedef internal::AmbiVector<double, int> AmbiVec; { @@ -769,6 +774,7 @@ // A coefficient that fails to construct leaves no node: the count of live nodes // the destructor walks may not include it, and the vector must remain usable. +template <typename = void> void test_ambivector_failed_insertion() { using ambivector_throwing::ThrowingScalar; typedef internal::AmbiVector<ThrowingScalar, int> AmbiVec; @@ -835,6 +841,7 @@ // next insertion relocates again instead of writing past its end. The leak // itself shows up under a leak checker; the retry below is what fails when the // capacity has already been advanced. +template <typename = void> void test_ambivector_failed_reallocation() { using ambivector_throwing::ThrowingScalar; typedef internal::AmbiVector<ThrowingScalar, int> AmbiVec; @@ -878,6 +885,7 @@ } #endif // EIGEN_EXCEPTIONS +template <typename = void> void test_sparse_vector_dense_product() { SparseVector<double> sv(3); sv.insert(0) = 1.0; @@ -894,7 +902,7 @@ sparse_structured_view_product_sfinae<SparseMatrix<double, RowMajor>>(); for (int i = 0; i < g_repeat; i++) { - CALL_SUBTEST_1((test_sparse_vector_dense_product())); + CALL_SUBTEST_1((test_sparse_vector_dense_product<>())); CALL_SUBTEST_1((sparse_product<SparseMatrix<double, ColMajor>>())); CALL_SUBTEST_1((sparse_product<SparseMatrix<double, RowMajor>>())); CALL_SUBTEST_1((bug_942<double>())); @@ -905,14 +913,14 @@ (sparse_product_regression_test<SparseMatrix<double, RowMajor>, Matrix<double, Dynamic, Dynamic, RowMajor>>())); CALL_SUBTEST_5((test_mixing_types<float>())); - CALL_SUBTEST_5((test_mixed_storage())); + CALL_SUBTEST_5((test_mixed_storage<>())); - CALL_SUBTEST_6((test_pruned_product_custom_scalar())); - CALL_SUBTEST_6((test_ambivector_discard_custom_scalar())); - CALL_SUBTEST_6((test_ambivector_resize_bounds())); + CALL_SUBTEST_6((test_pruned_product_custom_scalar<>())); + CALL_SUBTEST_6((test_ambivector_discard_custom_scalar<>())); + CALL_SUBTEST_6((test_ambivector_resize_bounds<>())); #if defined(EIGEN_EXCEPTIONS) - CALL_SUBTEST_6((test_ambivector_failed_insertion())); - CALL_SUBTEST_6((test_ambivector_failed_reallocation())); + CALL_SUBTEST_6((test_ambivector_failed_insertion<>())); + CALL_SUBTEST_6((test_ambivector_failed_reallocation<>())); #endif } }
diff --git a/test/sparseqr.cpp b/test/sparseqr.cpp index 09e4d8b..0d81774 100644 --- a/test/sparseqr.cpp +++ b/test/sparseqr.cpp
@@ -134,6 +134,7 @@ VERIFY_IS_APPROX(Q, dQ); } +template <typename = void> void test_sparseqr_factorize_uncompressed_input() { typedef SparseMatrix<double, ColMajor> MatrixType; typedef VectorXd Vector; @@ -191,6 +192,7 @@ VERIFY_IS_APPROX(denseA * x, b); } +template <typename = void> void test_sparseqr_lookahead_rejects_replaceable_weak_pivot() { Matrix<double, 4, 5> denseA; denseA << 10.875, 0.0, 0.0, 0.0, 0.0, -0.397597, 12.1403, 0.0, 0.0, 0.0317254, -0.851737, -0.0269339, 11.3113, @@ -201,6 +203,7 @@ verify_sparseqr_solves<COLAMDOrdering<int> >(denseA, b, 4); } +template <typename = void> void test_sparseqr_tiny_independent_column() { Matrix<double, 2, 3> denseA; Matrix<double, 2, 1> b; @@ -214,6 +217,7 @@ verify_sparseqr_solves<COLAMDOrdering<int> >(denseA, b, 2); } +template <typename = void> void test_sparseqr_lookahead_preserves_needed_weak_direction() { Matrix<double, 3, 4> denseA; Matrix<double, 3, 1> b; @@ -227,6 +231,7 @@ verify_sparseqr_solves<NaturalOrdering<int> >(denseA, b, 3); } +template <typename = void> void test_sparseqr_explicit_threshold_disables_lookahead() { typedef SparseMatrix<double, ColMajor> MatrixType; typedef SparseQR<MatrixType, NaturalOrdering<int> > Solver; @@ -308,11 +313,11 @@ CALL_SUBTEST_1(test_sparseqr_scalar<double>()); CALL_SUBTEST_2(test_sparseqr_scalar<std::complex<double> >()); } - CALL_SUBTEST_3(test_sparseqr_factorize_uncompressed_input()); - CALL_SUBTEST_4(test_sparseqr_lookahead_rejects_replaceable_weak_pivot()); - CALL_SUBTEST_5(test_sparseqr_tiny_independent_column()); - CALL_SUBTEST_6(test_sparseqr_explicit_threshold_disables_lookahead()); - CALL_SUBTEST_7(test_sparseqr_lookahead_preserves_needed_weak_direction()); + CALL_SUBTEST_3(test_sparseqr_factorize_uncompressed_input<>()); + CALL_SUBTEST_4(test_sparseqr_lookahead_rejects_replaceable_weak_pivot<>()); + CALL_SUBTEST_5(test_sparseqr_tiny_independent_column<>()); + CALL_SUBTEST_6(test_sparseqr_explicit_threshold_disables_lookahead<>()); + CALL_SUBTEST_7(test_sparseqr_lookahead_preserves_needed_weak_direction<>()); // The relative residual is a small multiple of the scalar's epsilon; bfloat16's coarser mantissa (7 bits vs // half's 10) loses more per operation, so it needs the larger factor despite the smaller system. CALL_SUBTEST_8(
diff --git a/test/special_members.cpp b/test/special_members.cpp index d3af8b2..bee25fb 100644 --- a/test/special_members.cpp +++ b/test/special_members.cpp
@@ -55,6 +55,7 @@ return moved; } +template <typename = void> void test_move_initialized_bdcsvd() { MatrixXd matrix(3, 3); matrix << 4.0, 1.0, 0.0, 1.0, 3.0, 1.0, 0.0, 1.0, 2.0; @@ -70,6 +71,7 @@ VERIFY_IS_APPROX(actual, expected); } +template <typename = void> void test_move_initialized_simplicial_llt() { const SparseMatrix<double> matrix = make_spd_matrix(); VectorXd expected(3); @@ -101,8 +103,8 @@ } EIGEN_DECLARE_TEST(special_members) { - CALL_SUBTEST_1(test_move_initialized_bdcsvd()); - CALL_SUBTEST_2(test_move_initialized_simplicial_llt()); + CALL_SUBTEST_1(test_move_initialized_bdcsvd<>()); + CALL_SUBTEST_2(test_move_initialized_simplicial_llt<>()); CALL_SUBTEST_3((test_move_initialized_iterative_solver<ConjugateGradient<SparseMatrix<double>, Lower | Upper>>())); CALL_SUBTEST_4((test_move_initialized_iterative_solver<BiCGSTAB<SparseMatrix<double>>>())); }
diff --git a/test/special_numbers.cpp b/test/special_numbers.cpp index c583d2b..def5ac9 100644 --- a/test/special_numbers.cpp +++ b/test/special_numbers.cpp
@@ -47,9 +47,74 @@ VERIFY(!mboth.array().allFinite()); } +// allFinite() over every position of a single non-finite coefficient, so the first packet, interior packets and the +// scalar tail are all reached, plus finite edge values that must not register. +template <typename Scalar> +void all_finite_positions() { + using RealScalar = typename NumTraits<Scalar>::Real; + using Vec = Matrix<Scalar, Dynamic, 1>; + RealScalar inf = std::numeric_limits<RealScalar>::infinity(); + RealScalar nan = std::numeric_limits<RealScalar>::quiet_NaN(); + RealScalar finite_edges[] = {RealScalar(0), + -RealScalar(0), + std::numeric_limits<RealScalar>::denorm_min(), + -std::numeric_limits<RealScalar>::denorm_min(), + NumTraits<RealScalar>::highest(), + NumTraits<RealScalar>::lowest()}; + RealScalar non_finite[] = {inf, -inf, nan}; + for (Index size : {Index(1), Index(3), Index(7), Index(16), Index(37)}) { + Vec v = Vec::Ones(size); + for (Index i = 0; i < size; ++i) v(i) = Scalar(finite_edges[i % 6]); + VERIFY(v.allFinite()); + for (Index i = 0; i < size; ++i) { + for (RealScalar x : non_finite) { + Vec w = v; + w(i) = Scalar(x); + VERIFY(!w.allFinite()); + VERIFY(!w.array().allFinite()); + } + } + } + // Row-major storage and an inner-strided map take the non-linear and strided visitor paths. + Matrix<Scalar, 5, 7, RowMajor> m = Matrix<Scalar, 5, 7, RowMajor>::Ones(); + VERIFY(m.allFinite()); + m(4, 6) = Scalar(nan); + VERIFY(!m.allFinite()); + VERIFY((m.template topLeftCorner<4, 6>().allFinite())); + Vec buffer = Vec::Ones(40); + Map<Vec, 0, InnerStride<2>> strided(buffer.data(), 20); + buffer(39) = Scalar(inf); + VERIFY(strided.allFinite()); + buffer(38) = Scalar(-inf); + VERIFY(!strided.allFinite()); +} + +template <typename Scalar> +void all_finite_complex() { + using RealScalar = typename NumTraits<Scalar>::Real; + RealScalar inf = std::numeric_limits<RealScalar>::infinity(); + RealScalar nan = std::numeric_limits<RealScalar>::quiet_NaN(); + Matrix<Scalar, Dynamic, 1> v = Matrix<Scalar, Dynamic, 1>::Ones(9); + VERIFY(v.allFinite()); + for (Index i = 0; i < v.size(); ++i) { + for (Scalar x : {Scalar(inf, 0), Scalar(0, -inf), Scalar(nan, 0), Scalar(0, nan)}) { + Matrix<Scalar, Dynamic, 1> w = v; + w(i) = x; + VERIFY(!w.allFinite()); + } + } +} + EIGEN_DECLARE_TEST(special_numbers) { for (int i = 0; i < 10 * g_repeat; i++) { CALL_SUBTEST_1(special_numbers<float>()); CALL_SUBTEST_1(special_numbers<double>()); } + CALL_SUBTEST_2(all_finite_positions<float>()); + CALL_SUBTEST_2(all_finite_positions<double>()); + CALL_SUBTEST_3(all_finite_positions<half>()); + CALL_SUBTEST_3(all_finite_positions<bfloat16>()); + CALL_SUBTEST_4(all_finite_complex<std::complex<float>>()); + CALL_SUBTEST_4(all_finite_complex<std::complex<double>>()); + CALL_SUBTEST_4(VERIFY((Matrix<int, Dynamic, 1>::Constant(17, (std::numeric_limits<int>::max)()).allFinite()))); }
diff --git a/test/spqr_support.cpp b/test/spqr_support.cpp index 5bda3f5..b3a6713 100644 --- a/test/spqr_support.cpp +++ b/test/spqr_support.cpp
@@ -55,6 +55,7 @@ VERIFY(x.isApprox(refX, test_precision<Scalar>())); } +template <typename = void> void test_spqr_fixed_ordering_uses_identity_permutation() { typedef SparseMatrix<double, ColMajor> MatrixType; typedef Matrix<double, Dynamic, Dynamic> DenseMatrix; @@ -94,6 +95,7 @@ VERIFY_IS_APPROX(x, refX); } +template <typename = void> void test_spqr_matrix_q_times_identity_expression() { typedef SparseMatrix<double, ColMajor> MatrixType; typedef Matrix<double, Dynamic, Dynamic> DenseMatrix; @@ -152,6 +154,6 @@ EIGEN_DECLARE_TEST(spqr_support) { CALL_SUBTEST_1(test_spqr_scalar<double>()); CALL_SUBTEST_2(test_spqr_scalar<std::complex<double> >()); - CALL_SUBTEST_3(test_spqr_fixed_ordering_uses_identity_permutation()); - CALL_SUBTEST_3(test_spqr_matrix_q_times_identity_expression()); + CALL_SUBTEST_3(test_spqr_fixed_ordering_uses_identity_permutation<>()); + CALL_SUBTEST_3(test_spqr_matrix_q_times_identity_expression<>()); }
diff --git a/test/stable_norm.cpp b/test/stable_norm.cpp index 519ccc9..a428c1a 100644 --- a/test/stable_norm.cpp +++ b/test/stable_norm.cpp
@@ -224,6 +224,7 @@ } } +template <typename = void> void test_empty() { Eigen::VectorXf empty(0); VERIFY_IS_EQUAL(empty.stableNorm(), 0.0f); @@ -495,6 +496,7 @@ VERIFY_IS_EQUAL(input.hypotNorm(), denorm); } +template <typename = void> void stable_norm_power_of_two_scaling() { // Reciprocal scaling rounds these norms two ULPs away from their correctly rounded values. Allow the fused and // non-fused reduction paths to differ by one ULP while still rejecting the previous results. @@ -582,6 +584,7 @@ VERIFY(abs(static_cast<float>(input(0).imag()) - 0.8f) <= tolerance); } +template <typename = void> void stable_normalize_promoted_factor() { typedef Matrix<half, Dynamic, 1> VectorX; using std::abs; @@ -603,6 +606,7 @@ VERIFY(abs(input.template cast<float>().norm() - 1.0f) <= tolerance); } +template <typename = void> void stable_normalize_no_malloc() { VectorXd input = VectorXd::Constant(2, (std::numeric_limits<double>::max)()); const Vector2d expected = Vector2d::Constant(1.0 / std::sqrt(2.0)); @@ -613,6 +617,7 @@ VERIFY_IS_APPROX(input, expected); } +template <typename = void> void stable_norm_expression_and_stride() { const Index size = 31; Index evaluation_count = 0; @@ -807,6 +812,7 @@ } } +template <typename = void> void stable_normalize_custom_scalar() { using Scalar = AnnoyingScalar; Matrix<Scalar, 2, 1> expected; @@ -825,7 +831,7 @@ } EIGEN_DECLARE_TEST(stable_norm) { - CALL_SUBTEST_1(test_empty()); + CALL_SUBTEST_1(test_empty<>()); for (int i = 0; i < g_repeat; i++) { CALL_SUBTEST_3(test_hypot<double>()); @@ -866,15 +872,15 @@ CALL_SUBTEST_11(stable_norm_mixed_underflow<double>()); CALL_SUBTEST_11(stable_norm_denormal_rounding<float>()); CALL_SUBTEST_11(stable_norm_denormal_rounding<double>()); - CALL_SUBTEST_11(stable_norm_power_of_two_scaling()); + CALL_SUBTEST_11(stable_norm_power_of_two_scaling<>()); CALL_SUBTEST_11(stable_norm_power_of_two_ftz<float>()); CALL_SUBTEST_11(stable_norm_power_of_two_ftz<double>()); CALL_SUBTEST_12(stable_norm_low_precision<half>()); CALL_SUBTEST_12(stable_norm_low_precision<bfloat16>()); CALL_SUBTEST_12(stable_norm_complex_low_precision<half>()); CALL_SUBTEST_12(stable_norm_complex_low_precision<bfloat16>()); - CALL_SUBTEST_12(stable_normalize_promoted_factor()); - CALL_SUBTEST_13(stable_norm_expression_and_stride()); - CALL_SUBTEST_13(stable_normalize_no_malloc()); - CALL_SUBTEST_13(stable_normalize_custom_scalar()); + CALL_SUBTEST_12(stable_normalize_promoted_factor<>()); + CALL_SUBTEST_13(stable_norm_expression_and_stride<>()); + CALL_SUBTEST_13(stable_normalize_no_malloc<>()); + CALL_SUBTEST_13(stable_normalize_custom_scalar<>()); }
diff --git a/test/stack_allocation_limit.cpp b/test/stack_allocation_limit.cpp index 3dc7971..fcae862 100644 --- a/test/stack_allocation_limit.cpp +++ b/test/stack_allocation_limit.cpp
@@ -29,4 +29,15 @@ for (Index i = 0; i < 4; ++i) VERIFY_IS_EQUAL(buffer[i], double(i)); } -EIGEN_DECLARE_TEST(stack_allocation_limit) { CALL_SUBTEST(check_stack_allocation_is_disabled()); } +// isApprox evaluates a lazy operand before its scaled fallback, which squares overflowing at 1e200 force; that +// temporary must not be a fixed-size object the limit rejects. +void check_approx_comparison_temporaries() { + using Vector = Matrix<double, 2, 1>; + VERIFY(Vector::Constant(1e200).isApprox(Vector::Constant(1e200))); + VERIFY(!Vector::Constant(1e200).isApprox(Vector::Constant(2e200))); +} + +EIGEN_DECLARE_TEST(stack_allocation_limit) { + CALL_SUBTEST(check_stack_allocation_is_disabled()); + CALL_SUBTEST(check_approx_comparison_temporaries()); +}
diff --git a/test/svd_common.h b/test/svd_common.h index a54b571..596d54b 100644 --- a/test/svd_common.h +++ b/test/svd_common.h
@@ -24,6 +24,32 @@ #include "svd_fill.h" #include "solverbase.h" +// U S V^H reconstructs m to the working precision plus the quantization of singular values stored in the subnormal +// range, checked 2^k above m so that FTZ/DAZ cannot flush the check itself: +// |U (2^k S) V^H - 2^k m|_max <= 64 n eps |2^k m|_max + n 2^k denorm_min, +// the second term because each singular value was rounded onto the subnormal grid, and its column carries that error +// into at most n entries with unit weights. The scaling by 2^k goes through the representation and is exact. +template <typename SvdType, typename MatrixType> +void svd_check_scaled_residual(const MatrixType& m, const SvdType& svd, int k) { + using Scalar = typename MatrixType::Scalar; + using RealScalar = typename MatrixType::RealScalar; + using RealVector = Matrix<RealScalar, Dynamic, 1>; + using DenseMatrix = Matrix<Scalar, Dynamic, Dynamic>; + const RealScalar eps = NumTraits<RealScalar>::epsilon(); + const Index diagSize = (std::min)(m.rows(), m.cols()); + const RealScalar n = RealScalar((std::max)(m.rows(), m.cols())); + const DenseMatrix scaledInput = m.unaryExpr(internal::scale_by_exponent_op<RealScalar>(k)); + const RealVector scaledSigma = svd.singularValues().unaryExpr(internal::scale_by_exponent_op<RealScalar>(k)); + const RealScalar granularity = numext::ldexp( + RealScalar(1), std::numeric_limits<RealScalar>::min_exponent - std::numeric_limits<RealScalar>::digits + k); + const DenseMatrix reconstruction = + svd.matrixU().leftCols(diagSize) * scaledSigma.asDiagonal() * svd.matrixV().leftCols(diagSize).adjoint(); + const RealScalar residual = (reconstruction - scaledInput).cwiseAbs().template maxCoeff<PropagateNaN>(); + const RealScalar tolerance = RealScalar(64) * n * eps * scaledInput.cwiseAbs().maxCoeff() + n * granularity; + VERIFY((numext::isfinite)(tolerance)); + VERIFY(residual <= tolerance); +} + // Check that the matrix m is properly reconstructed and that the U and V factors are unitary // The SVD must have already been computed. template <typename SvdType, typename MatrixType> @@ -44,7 +70,8 @@ MatrixVType v = svd.matrixV(); RealScalar scaling = m.cwiseAbs().maxCoeff(); if (scaling < (std::numeric_limits<RealScalar>::min)()) { - VERIFY(sigma.cwiseAbs().maxCoeff() <= (std::numeric_limits<RealScalar>::min)()); + // A subnormal or zero matrix: 2^(digits + 2) brings every nonzero entry into the normal range. + svd_check_scaled_residual(m, svd, std::numeric_limits<RealScalar>::digits + 2); } else { VERIFY_IS_APPROX(m / scaling, u * (sigma / scaling) * v.adjoint()); } @@ -249,6 +276,19 @@ VERIFY_IS_APPROX(x21, x3); } +// True when rank() dropped a positive singular value through its clamp at numeric_limits::min() alone, i.e. +// threshold() * sigma_0 <= sigma_rank < min(). Such a value need not be negligible against sigma_0, so +// m * solve(m * x) cannot reproduce m * x as check_solverbase requires; svd_least_square covers those inputs +// through the normal equations. An exactly zero singular value, as in the zero matrix, loses nothing. +template <typename SolverType> +bool svd_solve_truncated_by_clamp(const SolverType& solver) { + using RealScalar = typename SolverType::RealScalar; + const Index rank = solver.rank(); + if (rank == solver.singularValues().size()) return false; + const RealScalar dropped = solver.singularValues()(rank); + return dropped > RealScalar(0) && dropped >= solver.threshold() * solver.singularValues()(0); +} + template <typename MatrixType, typename SolverType> void svd_test_solvers(const MatrixType& m, const SolverType& solver) { Index rows, cols, cols2; @@ -261,10 +301,22 @@ } else { cols2 = cols; } + if (svd_solve_truncated_by_clamp(solver)) return; typedef Matrix<typename MatrixType::Scalar, MatrixType::ColsAtCompileTime, MatrixType::ColsAtCompileTime> CMatrixType; check_solverbase<CMatrixType, MatrixType>(m, solver, rows, cols, cols2); } +// The zero matrix has rank 0 without a positive singular value lost to the clamp, so its solves keep reproducing +// consistent right-hand sides. +template <typename MatrixType> +void svd_zero_matrix_solvers(Index rows, Index cols) { + const MatrixType m = MatrixType::Zero(rows, cols); + SVD_STATIC_OPTIONS(MatrixType, ComputeFullU | ComputeFullV) svd(m); + VERIFY(svd.rank() == 0); + VERIFY(!svd_solve_truncated_by_clamp(svd)); + svd_test_solvers(m, svd); +} + template <typename MatrixType, int Options, bool HasQrPreconditioner = (Options & internal::QRPreconditionerBits) != NoQRPreconditioner> struct svd_min_norm_if { @@ -332,6 +384,61 @@ VERIFY(svd.info() == InvalidInput); } +template <typename Scalar> +struct svd_subnormal_entry { + template <typename RealScalar> + static Scalar run(RealScalar real, RealScalar) { + return real; + } +}; + +template <typename RealScalar> +struct svd_subnormal_entry<std::complex<RealScalar>> { + static std::complex<RealScalar> run(RealScalar real, RealScalar imag) { return std::complex<RealScalar>(real, imag); } +}; + +// A rows-by-cols matrix whose entries are all subnormal: signed significands of digits - 2 bits times denorm_min, +// assembled without floating-point arithmetic so that FTZ/DAZ hardware cannot flush the fixture before the solver +// sees it. +template <typename MatrixType> +MatrixType svd_subnormal_fixture(Index rows, Index cols) { + using Scalar = typename MatrixType::Scalar; + using RealScalar = typename MatrixType::RealScalar; + using Binary = internal::binary_floating_point_traits<RealScalar>; + using Bits = typename Binary::Bits; + const Bits top = Bits(1) << (std::numeric_limits<RealScalar>::digits - 3); + const auto component = [&]() { + const Bits sign = internal::random<bool>() ? Binary::kSignBit : Bits(0); + return numext::bit_cast<RealScalar>(sign | (top + internal::random<Bits>(Bits(0), top - Bits(1)))); + }; + MatrixType m(rows, cols); + for (Index j = 0; j < cols; ++j) + for (Index i = 0; i < rows; ++i) m(i, j) = svd_subnormal_entry<Scalar>::run(component(), component()); + return m; +} + +// svd decomposed the all-subnormal matrix m, scaled (computed beforehand, under gradual underflow) its power-of-two +// multiple ms = m * 2^k. Both are normalized by a power of two, so they run the same iteration on work matrices a +// power of two apart and their singular values agree bit for bit up to the rounding of svd's onto the subnormal grid: +// |2^k sigma_i - sigma_i(ms)| <= 2^k denorm_min / 2. +// U and V then reconstruct m to the scaled residual bound. +template <typename SvdType, typename MatrixType> +void svd_check_flushed_subnormal(const SvdType& svd, const SvdType& scaled, const MatrixType& m, int k) { + using RealScalar = typename MatrixType::RealScalar; + using RealVector = Matrix<RealScalar, Dynamic, 1>; + VERIFY_IS_EQUAL(svd.info(), Success); + VERIFY_IS_EQUAL(scaled.info(), Success); + VERIFY_IS_EQUAL(svd.nonzeroSingularValues(), scaled.nonzeroSingularValues()); + const RealVector sigmaUp = svd.singularValues().unaryExpr(internal::scale_by_exponent_op<RealScalar>(k)); + VERIFY(scaled.singularValues()(0) > RealScalar(0)); + const RealScalar halfGranularity = numext::ldexp( + RealScalar(1), std::numeric_limits<RealScalar>::min_exponent - std::numeric_limits<RealScalar>::digits + k - 1); + VERIFY((sigmaUp - scaled.singularValues()).cwiseAbs().template maxCoeff<PropagateNaN>() <= halfGranularity); + VERIFY_IS_UNITARY(svd.matrixU()); + VERIFY_IS_UNITARY(svd.matrixV()); + svd_check_scaled_residual(m, svd, k); +} + // Regression test for bug 286: JacobiSVD loops indefinitely with some // matrices containing denormal numbers. template <typename> @@ -570,6 +677,63 @@ VERIFY_RAISES_ASSERT(fullSvd.compute(dynamicMatrix)); } +// For A = Q D with Q unitary and D diagonal, column-pivoted QR gives R = D up to rounding, so the Jacobi sweep on R +// recovers every singular value to high relative accuracy: +// |sigma_i - d_i| / d_i <= 4 n eps. +// For n below BDCSVD's switch size, BDCSVD will invoke JacobiSVD. +template <typename MatrixType> +void svd_precondition_square_matrix_accuracy(Index n) { + using Scalar = typename MatrixType::Scalar; + using RealScalar = typename MatrixType::RealScalar; + // The scaling below spans [1e-12, 1] only for n >= 2. + VERIFY(n >= 2); + VectorX<RealScalar> scaling(n); + for (Index i = 0; i < n; ++i) scaling(i) = RealScalar(std::pow(1e12, -double(i) / double(n - 1))); + const MatrixType m = generateRandomUnitaryMatrix<MatrixType>(n) * scaling.template cast<Scalar>().asDiagonal(); + + const VectorX<RealScalar> singularValues = + SVD_STATIC_OPTIONS(MatrixType, PreconditionSquareMatrix)(m).singularValues(); + const RealScalar bound = RealScalar(4 * n) * NumTraits<RealScalar>::epsilon(); + VERIFY(((singularValues - scaling).array().abs() / scaling.array()).template maxCoeff<PropagateNaN>() <= bound); +} + +// PreconditionSquareMatrix in Options must not change what the deprecated runtime options request: U and V are computed +// exactly when their bits are set, thin or full as requested. +template <typename SvdType, typename MatrixType> +void svd_check_runtime_options_match(const MatrixType& m, const SvdType& svd, unsigned int options) { + const Index diagSize = (std::min)(m.rows(), m.cols()); + VERIFY_IS_EQUAL(svd.computeU(), (options & (ComputeThinU | ComputeFullU)) != 0); + VERIFY_IS_EQUAL(svd.computeV(), (options & (ComputeThinV | ComputeFullV)) != 0); + if (svd.computeU()) { + VERIFY_IS_EQUAL(svd.matrixU().rows(), m.rows()); + VERIFY_IS_EQUAL(svd.matrixU().cols(), (options & ComputeThinU) ? diagSize : m.rows()); + VERIFY(svd.matrixU().isUnitary()); + } + if (svd.computeV()) { + VERIFY_IS_EQUAL(svd.matrixV().rows(), m.cols()); + VERIFY_IS_EQUAL(svd.matrixV().cols(), (options & ComputeThinV) ? diagSize : m.cols()); + VERIFY(svd.matrixV().isUnitary()); + } + if (svd.computeU() && svd.computeV()) svd_check_scaled_residual(m, svd, 0); +} + +EIGEN_DIAGNOSTICS(push) +EIGEN_DISABLE_DEPRECATED_WARNING +template <typename MatrixType> +void svd_precondition_square_matrix_runtime_options(Index size) { + MatrixType m(size, size); + svd_fill_random(m); + for (unsigned int options : {0u, unsigned(ComputeThinU | ComputeThinV), unsigned(ComputeFullU | ComputeFullV), + unsigned(ComputeFullU), unsigned(ComputeThinV)}) { + const SVD_STATIC_OPTIONS(MatrixType, PreconditionSquareMatrix) constructed(m, options); + svd_check_runtime_options_match(m, constructed, options); + SVD_STATIC_OPTIONS(MatrixType, PreconditionSquareMatrix) computed; + computed.compute(m, options); + svd_check_runtime_options_match(m, computed, options); + } +} +EIGEN_DIAGNOSTICS(pop) + #undef SVD_DEFAULT #undef SVD_FOR_MIN_NORM #undef SVD_STATIC_OPTIONS
diff --git a/test/triangular.cpp b/test/triangular.cpp index ce00fae..132cbfe 100644 --- a/test/triangular.cpp +++ b/test/triangular.cpp
@@ -38,6 +38,16 @@ internal::void_t<decltype(std::declval<const ViewType&>() + std::declval<const ViewType&>())>> : std::true_type {}; +template <typename ViewType, typename DiagonalType, typename = void> +struct has_diagonal_sum : std::false_type {}; + +template <typename ViewType, typename DiagonalType> +struct has_diagonal_sum< + ViewType, DiagonalType, + internal::void_t<decltype(std::declval<const ViewType&>() + std::declval<const DiagonalType&>()), + decltype(std::declval<const DiagonalType&>() - std::declval<const ViewType&>())>> + : std::true_type {}; + template <unsigned int Mode, typename MatrixType> void triangular_scalar_multiply(const MatrixType& m) { typedef typename MatrixType::Scalar Scalar; @@ -110,6 +120,55 @@ selfadjoint_structured_sum_impl<Mode, MatrixType>::run(s, a, b, result, reference); } +// Permutation times triangular or self-adjoint view, both orders, with P and P^-1. Rows or columns of the +// densified view are only moved, so the products are exact. +template <typename ViewType, typename PermutationType> +void triangular_permutation_products_check(const TriangularBase<ViewType>& view, const PermutationType& perm) { + using Scalar = typename ViewType::Scalar; + using RealScalar = typename NumTraits<Scalar>::Real; + using MatrixType = typename ViewType::DenseMatrixType; + + const MatrixType dense = view.derived().toDenseMatrix(); + const MatrixType permDense = perm.toDenseMatrix().template cast<Scalar>(); + const MatrixType inverseDense = perm.inverse().toDenseMatrix().template cast<Scalar>(); + + MatrixType result = perm * view.derived(); + VERIFY_IS_EQUAL(result, MatrixType(permDense * dense)); + result = view.derived() * perm; + VERIFY_IS_EQUAL(result, MatrixType(dense * permDense)); + result = perm.inverse() * view.derived(); + VERIFY_IS_EQUAL(result, MatrixType(inverseDense * dense)); + result = view.derived() * perm.transpose(); + VERIFY_IS_EQUAL(result, MatrixType(dense * inverseDense)); + result = perm * view.derived() * perm.inverse(); + VERIFY_IS_APPROX(result, MatrixType(permDense * dense * inverseDense)); + + // alpha cannot fold into P or into a unit diagonal, so alpha * (P * V) and alpha * (UnitTri * P^-1) evaluate the + // product first and scale it afterwards. A real alpha keeps the scaling exact on every evaluation path. + const Scalar alpha(internal::random<RealScalar>(1, 2)); + result = alpha * (perm * view.derived()); + VERIFY_IS_EQUAL(result, MatrixType(alpha * MatrixType(permDense * dense))); + result = alpha * (view.derived() * perm.inverse()); + VERIFY_IS_EQUAL(result, MatrixType(alpha * MatrixType(dense * inverseDense))); + result.setZero(); + result.noalias() += alpha * (perm.inverse() * view.derived()); + VERIFY_IS_EQUAL(result, MatrixType(alpha * MatrixType(inverseDense * dense))); +} + +template <typename MatrixType> +void triangular_permutation_products(const MatrixType& m) { + PermutationMatrix<MatrixType::RowsAtCompileTime> perm(m.rows()); + randomPermutationVector(perm.indices(), m.rows()); + const MatrixType a = MatrixType::Random(m.rows(), m.cols()); + + triangular_permutation_products_check(a.template triangularView<Upper>(), perm); + triangular_permutation_products_check(a.template triangularView<Lower>(), perm); + triangular_permutation_products_check(a.template triangularView<UnitLower>(), perm); + triangular_permutation_products_check(a.template triangularView<StrictlyUpper>(), perm); + triangular_permutation_products_check(a.template selfadjointView<Lower>(), perm); + triangular_permutation_products_check(a.template selfadjointView<Upper>(), perm); +} + template <typename MatrixType> void triangular_setters(const MatrixType& m) { const Index rows = m.rows(); @@ -165,6 +224,49 @@ STATIC_CHECK((has_structured_sum<StrictlyLowerView>::value)); STATIC_CHECK((has_structured_sum<StrictlyUpperView>::value)); STATIC_CHECK((!has_structured_sum<UnitLowerView>::value)); + + // Sums with a diagonal matrix exist only for modes that store their diagonal. + typedef DiagonalMatrix<typename MatrixType::Scalar, MatrixType::RowsAtCompileTime> DiagonalType; + typedef decltype(std::declval<MatrixType&>().template selfadjointView<Lower>()) SelfAdjointLowerView; + STATIC_CHECK((has_diagonal_sum<UpperView, DiagonalType>::value)); + STATIC_CHECK((has_diagonal_sum<LowerView, DiagonalType>::value)); + STATIC_CHECK((has_diagonal_sum<SelfAdjointLowerView, DiagonalType>::value)); + STATIC_CHECK((!has_diagonal_sum<StrictlyLowerView, DiagonalType>::value)); + STATIC_CHECK((!has_diagonal_sum<StrictlyUpperView, DiagonalType>::value)); + STATIC_CHECK((!has_diagonal_sum<UnitLowerView, DiagonalType>::value)); +} + +// view +/- diagonal keeps the view's mode; the dense value is the view of the dense sum. +template <unsigned int Mode, typename MatrixType> +void triangular_diagonal_sum(const MatrixType& m) { + typedef typename MatrixType::Scalar Scalar; + typedef Matrix<Scalar, MatrixType::RowsAtCompileTime, 1> VectorType; + + const Index rows = m.rows(); + const MatrixType a = MatrixType::Random(rows, rows); + const VectorType d = VectorType::Random(rows); + const auto view = internal::make_triangular_base_cwise_view<Mode>(a); + const MatrixType viewDense = view.toDenseMatrix(); + const MatrixType diagDense = d.asDiagonal().toDenseMatrix(); + + STATIC_CHECK((int(std::decay_t<decltype(view + d.asDiagonal())>::Mode) == int(Mode))); + STATIC_CHECK((int(std::decay_t<decltype(d.asDiagonal() - view)>::Mode) == int(Mode))); + + MatrixType result = view + d.asDiagonal(); + VERIFY_IS_APPROX(result, viewDense + diagDense); + result = d.asDiagonal() + view; + VERIFY_IS_APPROX(result, viewDense + diagDense); + result = view - d.asDiagonal(); + VERIFY_IS_APPROX(result, viewDense - diagDense); + result = d.asDiagonal() - view; + VERIFY_IS_APPROX(result, diagDense - viewDense); + result = (view + d.asDiagonal()).toDenseMatrix(); + VERIFY_IS_APPROX(result, viewDense + diagDense); + + // The dense operand of the lazy sum is the view's nested expression: its other triangle must not leak. + const DiagonalMatrix<Scalar, MatrixType::RowsAtCompileTime> dm(d); + result = view + dm; + VERIFY_IS_APPROX(result, viewDense + diagDense); } template <typename MatrixType> @@ -201,6 +303,11 @@ triangular_structured_sum<Lower>(m); triangular_structured_sum<StrictlyUpper>(m); triangular_structured_sum<StrictlyLower>(m); + triangular_permutation_products(m); + triangular_diagonal_sum<Upper>(m); + triangular_diagonal_sum<Lower>(m); + triangular_diagonal_sum<Upper | SelfAdjoint>(m); + triangular_diagonal_sum<Lower | SelfAdjoint>(m); triangular_setters(m); RealScalar largerEps = 10 * test_precision<RealScalar>();
diff --git a/test/tridiagonal_eigensolver.cpp b/test/tridiagonal_eigensolver.cpp index 1ddb1a9..29b23c2 100644 --- a/test/tridiagonal_eigensolver.cpp +++ b/test/tridiagonal_eigensolver.cpp
@@ -9,6 +9,7 @@ // SPDX-License-Identifier: MPL-2.0 #include "main.h" +#include "fp_control.h" #include "tridiag_test_matrices.h" #include <limits> #include <Eigen/Eigenvalues> @@ -628,6 +629,7 @@ } } +template <typename = void> void tridiagonal_eigensolver_power_of_two_scaling() { // An input sensitive to normalization rounding; bisection promises normwise accuracy, not correct rounding. Vector3f diag; @@ -740,12 +742,88 @@ } } +// d and e against their power-of-two scaled copies, computed beforehand, in every flush-to-zero mode: eigenvalues to +// the bisection error plus the quantization of the range they are stored in, eigenvectors (invariant under the +// scaling) against the scaled residual. +template <typename RealScalar> +void tridiagonal_check_scaled_pair(const Matrix<RealScalar, Dynamic, 1>& d, const Matrix<RealScalar, Dynamic, 1>& e, + int k) { + using MatrixType = Matrix<RealScalar, Dynamic, Dynamic>; + using VectorType = Matrix<RealScalar, Dynamic, 1>; + const RealScalar eps = NumTraits<RealScalar>::epsilon(); + constexpr int digits = std::numeric_limits<RealScalar>::digits; + const Index n = d.size(); + const VectorType ds = d.unaryExpr(internal::scale_by_exponent_op<RealScalar>(k)); + const VectorType es = e.unaryExpr(internal::scale_by_exponent_op<RealScalar>(k)); + const MatrixType Ts = dense_symmetric_tridiag(ds, es); + TridiagonalEigenSolver<RealScalar> ref; + ref.compute(ds, es); + VERIFY_IS_EQUAL(ref.info(), Success); + + forEachFlushToZeroMode([&](FlushToZeroMode) { + TridiagonalEigenSolver<RealScalar> sub; + sub.compute(d, e); + VERIFY_IS_EQUAL(sub.info(), Success); + const VectorType up = sub.eigenvalues().unaryExpr(internal::scale_by_exponent_op<RealScalar>(k)); + const RealScalar radius = ref.eigenvalues().cwiseAbs().maxCoeff(); + // denorm_min 2^k: the spacing of the subnormal range the eigenvalues were stored in, scaled up with them. + const RealScalar granularity = + numext::ldexp(RealScalar(1), std::numeric_limits<RealScalar>::min_exponent - digits + k); + // Named before the checks: MSVC does not capture a local that a lambda first uses after a `.template` call in the + // same expression (C2065). + const RealScalar valueBound = RealScalar(16) * (RealScalar(n) * eps * radius + granularity); + const RealScalar residualBound = RealScalar(64) * RealScalar(n) * eps * radius; + VERIFY((up - ref.eigenvalues()).cwiseAbs().template maxCoeff<PropagateNaN>() <= valueBound); + const MatrixType V = sub.eigenvectors(); + VERIFY_IS_UNITARY(V); + VERIFY((Ts * V - V * ref.eigenvalues().asDiagonal()).cwiseAbs().template maxCoeff<PropagateNaN>() <= residualBound); + }); +} + +// All-subnormal d and e, assembled from significands without floating-point arithmetic, which a SIMD unit that +// flushes subnormal inputs (ARMv7 NEON, Arm FZ, DAZ) reads as zero in the maximum that selects the normalization. +template <typename RealScalar> +void tridiagonal_eigensolver_flushed_subnormal(Index n) { + using VectorType = Matrix<RealScalar, Dynamic, 1>; + using Binary = internal::binary_floating_point_traits<RealScalar>; + using Bits = typename Binary::Bits; + constexpr int digits = std::numeric_limits<RealScalar>::digits; + // Distinct diagonal significands of digits - 2 bits, off-diagonal ones of alternating sign. + const Bits top = Bits(1) << (digits - 3); + VectorType d(n), e(n - 1); + for (Index i = 0; i < n; ++i) d(i) = numext::bit_cast<RealScalar>(top + Bits(i) * (top / Bits(16))); + for (Index i = 0; i < n - 1; ++i) { + const Bits sign = (i % 2 == 0) ? Bits(0) : Binary::kSignBit; + e(i) = numext::bit_cast<RealScalar>(sign | (top / Bits(2) + Bits(i) * (top / Bits(32)))); + } + tridiagonal_check_scaled_pair(d, e, 60); +} + +// A normal maximum below min / eps with subnormal couplings: |e| flushed to zero would split the matrix into 1 x 1 +// blocks with eigenvectors e_i, although the true ones are (1, +-1) / sqrt(2) for equal d. +template <typename RealScalar> +void tridiagonal_eigensolver_flushed_subnormal_coupling() { + using VectorType = Matrix<RealScalar, Dynamic, 1>; + constexpr int minExponent = std::numeric_limits<RealScalar>::min_exponent; + VectorType d(4), e(3); + d.setConstant(numext::ldexp(RealScalar(1), minExponent + 5)); + for (Index i = 0; i < 3; ++i) e(i) = numext::ldexp(RealScalar(1) + RealScalar(i) / RealScalar(4), minExponent - 4); + tridiagonal_check_scaled_pair(d, e, 40); +} + EIGEN_DECLARE_TEST(tridiagonal_eigensolver) { CALL_SUBTEST_1(tridiagonal_eigensolver_subnormal_staged<double>()); CALL_SUBTEST_2(tridiagonal_eigensolver_subnormal_staged<float>()); - CALL_SUBTEST_2(tridiagonal_eigensolver_power_of_two_scaling()); + CALL_SUBTEST_2(tridiagonal_eigensolver_power_of_two_scaling<>()); CALL_SUBTEST_2(tridiagonal_eigensolver_scaling_units<float>()); CALL_SUBTEST_1(tridiagonal_eigensolver_scaling_units<double>()); + for (Index n : {Index(1), Index(2), Index(9)}) { + EIGEN_UNUSED_VARIABLE(n); // parts 3-6 compile no subtest that takes n + CALL_SUBTEST_1(tridiagonal_eigensolver_flushed_subnormal<double>(n)); + CALL_SUBTEST_2(tridiagonal_eigensolver_flushed_subnormal<float>(n)); + } + CALL_SUBTEST_1(tridiagonal_eigensolver_flushed_subnormal_coupling<double>()); + CALL_SUBTEST_2(tridiagonal_eigensolver_flushed_subnormal_coupling<float>()); for (int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(tridiagonal_eigensolver_bisection<double>()); CALL_SUBTEST_2(tridiagonal_eigensolver_bisection<float>());
diff --git a/test/trsm_packet_row_major_default.cpp b/test/trsm_packet_row_major_default.cpp new file mode 100644 index 0000000..4955ae6 --- /dev/null +++ b/test/trsm_packet_row_major_default.cpp
@@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +// EIGEN_SUFFIXES;1;2 +// The packet kernel's views of the column-major right-hand side must keep their +// storage order when EIGEN_DEFAULT_TO_ROW_MAJOR flips the default. +#ifndef EIGEN_DEFAULT_TO_ROW_MAJOR +#define EIGEN_DEFAULT_TO_ROW_MAJOR +#endif +#include "trsm_packet.cpp" // NOLINT(bugprone-suspicious-include): Compile the suite with a different configuration.
diff --git a/test/unary_pow.cpp b/test/unary_pow.cpp new file mode 100644 index 0000000..31c4534 --- /dev/null +++ b/test/unary_pow.cpp
@@ -0,0 +1,586 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 + +#include "main.h" + +#include <cfenv> +#include <cmath> +#include <complex> +#include <limits> +#include <vector> + +// use_double_word<packet_traits<Scalar>::type> names a vector type such as __m512d as a template argument. +#if EIGEN_GNUC_STRICT_AT_LEAST(6, 0, 0) +#pragma GCC diagnostic ignored "-Wignored-attributes" +#endif + +// Integer powers of binary32 and binary64 bases, real and complex, are computed by repeated squaring in +// double-word arithmetic: correctly rounded up to the rare cases where the exact power lies within about +// 7 * n * u^2 of a rounding boundary. These tests check that bound against independent references, on the +// vectorized path and on the scalar path, together with the exceptional values and signed zeros. + +template <typename Scalar> +Scalar ulp_of(const Scalar& x) { + Scalar a = numext::abs(x); + return numext::nextafter(a, std::numeric_limits<Scalar>::infinity()) - a; +} + +// |a - b| in units of ulp(b), with non-finite and zero references matched exactly (including the sign of zero). +template <typename Scalar> +bool within_ulps(const Scalar& a, const Scalar& b, double ulps) { + if ((numext::isnan)(a) || (numext::isnan)(b)) return (numext::isnan)(a) && (numext::isnan)(b); + if ((numext::isinf)(b) || b == Scalar(0)) return a == b && bool(numext::signbit(a)) == bool(numext::signbit(b)); + if ((numext::isinf)(a)) return false; + return double(numext::abs(a - b)) <= ulps * double(ulp_of(b)); +} + +// Log-uniform magnitudes over [2^lo, 2^hi], both signs. +template <typename Scalar> +std::vector<Scalar> log_uniform_bases(int lo, int hi, int count) { + std::vector<Scalar> bases; + for (int i = 0; i < count; ++i) { + double exponent = internal::random<double>(double(lo), double(hi)); + Scalar x = Scalar(std::ldexp(internal::random<double>(1.0, 2.0), int(std::floor(exponent)))); + bases.push_back(i % 2 ? -x : x); + } + return bases; +} + +// y = x.pow(n), returning which of FE_INVALID and FE_OVERFLOW it raised on a target whose packets flush subnormals, +// where the results below the smallest normal are rebuilt from integer bits. Elsewhere it reports none: the double +// word can raise FE_INVALID for a zero base, which an AVX2 partial packet holds as padding. +template <typename ArrayType, typename Exponent> +int pow_exceptions(const ArrayType& x, const Exponent& n, ArrayType& y) { + using Packet = typename internal::packet_traits<typename ArrayType::Scalar>::type; + std::fenv_t saved_environment; + const bool environment_saved = + internal::unary_pow::flushes_subnormals<Packet>::value && std::feholdexcept(&saved_environment) == 0; + y = x.pow(n); + const int exceptions = environment_saved ? std::fetestexcept(FE_INVALID | FE_OVERFLOW) : 0; + if (environment_saved) std::fesetenv(&saved_environment); + return exceptions; +} + +// Reference: pow in the next wider format, so its own rounding is negligible for float; for double, the +// platform pow, which is within 1 ulp on every supported libm. +template <typename Scalar> +Scalar reference_pow(const Scalar& x, long n) { + using Wide = std::conditional_t<std::is_same<Scalar, float>::value, double, Scalar>; + return Scalar(std::pow(Wide(x), Wide(n))); +} + +template <typename Scalar, typename Exponent> +void check_real_pow(const std::vector<Scalar>& bases, const std::vector<long>& exponents, double ulps) { + Index size = 2 * internal::packet_traits<Scalar>::size + 1; + ArrayX<Scalar> x(size), y(size); + for (long n : exponents) { + Exponent exponent = Exponent(n); + for (const Scalar& base : bases) { + x.setConstant(base); + y = x.pow(exponent); + Scalar reference = reference_pow(base, n); + for (Index i = 0; i < size; ++i) { + bool ok = within_ulps(y(i), reference, ulps); + if (!ok) + std::cout << "pow(" << base << ", " << n << ") = " << y(i) << " != " << reference << " (" << ulps << " ulp)" + << std::endl; + VERIFY(ok); + } + } + } +} + +template <typename Scalar> +void real_pow_test() { + // Exponents on both sides of the squaring cutoffs: the plain-squaring range [-3, 7] of other scalar types, the + // fallback to generic_pow above 2^12 (float) or 2^20 (double), and integer exponents beyond the significand, + // which generic_pow could not represent. + long cutoff = long(internal::unary_pow::max_squaring_exponent<Scalar>()); + const std::vector<long> exponents = {2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 100, 127, 1000, + -2, -3, -4, -5, -7, -8, -9, -15, -100, -127, -1000, 4096, cutoff, -cutoff}; + const std::vector<long> huge_exponents = {cutoff + 1, + -cutoff - 1, + 1 << 24, + -(1 << 24), + (1 << 24) + 1, + -((1 << 24) + 1), + (std::numeric_limits<int>::max)(), + (std::numeric_limits<int>::min)()}; + constexpr int min_exponent = std::numeric_limits<Scalar>::min_exponent; + constexpr int max_exponent = std::numeric_limits<Scalar>::max_exponent; + + // Correctly rounded within 2 ulps only holds for double-word repeated squaring, whose fallback -- plain + // squaring -- is just "a few ulps" and exercised elsewhere. Only the MSA and HVX packets lack the path, so + // require it everywhere else rather than skipping on the trait that selects it: losing it has to fail here. + using Packet = typename internal::packet_traits<Scalar>::type; + constexpr bool has_double_word = internal::unary_pow::use_double_word<Packet>::value; +#if !defined(EIGEN_VECTORIZE_MSA) && !defined(EIGEN_VECTORIZE_HVX) + VERIFY(has_double_word); +#endif + if (has_double_word) { + for (long n : exponents) { + int limit = numext::mini((max_exponent - 2) / int(numext::abs(n)), max_exponent - 2); + const std::vector<Scalar> bases = log_uniform_bases<Scalar>(-limit, limit, 64); + check_real_pow<Scalar, int>(bases, {n}, 2.0); + check_real_pow<Scalar, Scalar>(bases, {n}, 2.0); + } + } + // Results near the underflow and overflow thresholds: the final scaling rounds once more into the subnormal + // range, and the reference rounds too. + { + const std::vector<Scalar> bases = log_uniform_bases<Scalar>(min_exponent - 4, 4, 32); + check_real_pow<Scalar, int>(bases, {2, 3, 8, -2, -3, -8}, 3.0); + check_real_pow<Scalar, Scalar>(bases, {2, 3, 8, -2, -3, -8}, 3.0); + const std::vector<Scalar> tiny = log_uniform_bases<Scalar>(min_exponent - 20, min_exponent / 8, 32); + check_real_pow<Scalar, int>(tiny, {8, 9, 100, -8, -9, -100}, 3.0); + check_real_pow<Scalar, Scalar>(tiny, {8, 9, 100, -8, -9, -100}, 3.0); + } + // x^-1 and x^2 round once, as 1/x and x * x do, also where their results are subnormal. + { + const Index size = 2 * internal::packet_traits<Scalar>::size + 1; + for (const Scalar& base : log_uniform_bases<Scalar>(max_exponent - 3, max_exponent, 64)) { + ArrayX<Scalar> y = ArrayX<Scalar>::Constant(size, base).pow(-1); + for (Index k = 0; k < size; ++k) VERIFY(within_ulps(y(k), Scalar(1) / base, 0.0)); + } + for (const Scalar& base : log_uniform_bases<Scalar>(min_exponent / 2 - 13, min_exponent / 2, 64)) { + ArrayX<Scalar> y = ArrayX<Scalar>::Constant(size, base).pow(2); + for (Index k = 0; k < size; ++k) VERIFY(within_ulps(y(k), base * base, 0.0)); + } + } + // A subnormal power beside normal ones, 2^-n and 1^n in one packet, raises neither FE_INVALID nor FE_OVERFLOW. + { + const int n = 1 - min_exponent + std::numeric_limits<Scalar>::digits / 2; + const Scalar tiny = Scalar(std::ldexp(1.0, -n)); + ArrayX<Scalar> x = ArrayX<Scalar>::Ones(2 * internal::packet_traits<Scalar>::size + 1), y; + x(0) = Scalar(0.5); + VERIFY_IS_EQUAL(pow_exceptions(x, n, y), 0); + for (Index k = 0; k < x.size(); ++k) VERIFY(within_ulps(y(k), k == 0 ? tiny : Scalar(1), 0.0)); + VERIFY_IS_EQUAL(pow_exceptions(x, Scalar(n), y), 0); + for (Index k = 0; k < x.size(); ++k) VERIFY(within_ulps(y(k), k == 0 ? tiny : Scalar(1), 0.0)); + } + // Exponents beyond the squaring cutoff, on bases within a few ulps of one, where the power stays finite for + // every exponent up to INT_MAX. The reference pow then carries the full rounding error of its own exponent + // conversion, which is why the double reference is allowed a wider margin. + { + std::vector<Scalar> bases; + for (int i = -16; i <= 16; ++i) bases.push_back(Scalar(1) + Scalar(i) * NumTraits<Scalar>::epsilon()); + check_real_pow<Scalar, int>(bases, huge_exponents, 3.0); + for (long n : huge_exponents) + if (n == long(Scalar(n))) check_real_pow<Scalar, Scalar>(bases, {n}, 3.0); + } + // Exceptional values and signed zeros, exactly as std::pow. + { + Scalar inf = std::numeric_limits<Scalar>::infinity(); + Scalar nan = std::numeric_limits<Scalar>::quiet_NaN(); + Scalar subnormal = std::numeric_limits<Scalar>::denorm_min() * Scalar(3); + const std::vector<Scalar> specials = {Scalar(0), + -Scalar(0), + inf, + -inf, + nan, + subnormal, + -subnormal, + (std::numeric_limits<Scalar>::max)(), + (std::numeric_limits<Scalar>::min)(), + Scalar(1), + Scalar(-1)}; + const std::vector<long> small = {1, 2, 3, 4, 5, 8, 9, 100, -1, -2, -3, -4, -5, -8, -9, -100}; + check_real_pow<Scalar, int>(specials, small, 1.0); + check_real_pow<Scalar, Scalar>(specials, small, 1.0); + } +} + +// Complex reference: componentwise double-word repeated multiplication, no scaling, so the bases are kept where +// nothing under- or overflows. +template <typename Real> +struct double_word { + Real hi, lo; + static double_word product(const double_word& a, const double_word& b) { + Real p = a.hi * b.hi; + Real e = numext::fma(a.hi, b.hi, -p) + (a.hi * b.lo + a.lo * b.hi); + Real s = p + e; + return {s, e - (s - p)}; + } + static double_word sum(const double_word& a, const double_word& b) { + Real s = a.hi + b.hi; + Real bb = s - a.hi; + Real e = ((a.hi - (s - bb)) + (b.hi - bb)) + a.lo + b.lo; + Real t = s + e; + return {t, e - (t - s)}; + } + static double_word negate(const double_word& a) { return {-a.hi, -a.lo}; } +}; + +template <typename Real> +std::complex<Real> reference_complex_pow(const std::complex<Real>& z, long n) { + using DW = double_word<Real>; + DW re{Real(1), Real(0)}, im{Real(0), Real(0)}; + DW zr{numext::real(z), Real(0)}, zi{numext::imag(z), Real(0)}; + for (long k = 0; k < numext::abs(n); ++k) { + DW new_re = DW::sum(DW::product(re, zr), DW::negate(DW::product(im, zi))); + im = DW::sum(DW::product(re, zi), DW::product(im, zr)); + re = new_re; + } + if (n < 0) { + // 1/(re + im i) = (re - im i)/(re^2 + im^2) in double-word arithmetic, then rounded. + using LongDouble = long double; + DW norm = DW::sum(DW::product(re, re), DW::product(im, im)); + LongDouble inv = 1.0L / (LongDouble(norm.hi) + LongDouble(norm.lo)); + return std::complex<Real>(Real((LongDouble(re.hi) + LongDouble(re.lo)) * inv), + Real(-(LongDouble(im.hi) + LongDouble(im.lo)) * inv)); + } + return std::complex<Real>(Real(re.hi + re.lo), Real(im.hi + im.lo)); +} + +// The reciprocal above rounds in long double, which is a reference only where that is wider than Real: not where +// long double is binary64 (MSVC, Apple arm64), for Real = double. +template <typename Real> +bool has_complex_reference(long n) { + return n >= 0 || sizeof(long double) > sizeof(Real); +} + +// The error of a complex result is measured against the magnitude of the reference: a component that nearly +// cancels is not required to be accurate on its own. +template <typename Real> +bool complex_within_ulps(const std::complex<Real>& a, const std::complex<Real>& b, double ulps) { + double magnitude = double(numext::abs(b)); + double error = double(numext::abs(a - b)); + return error <= ulps * double(ulp_of(Real(magnitude))); +} + +template <typename Real, typename Exponent> +void check_complex_pow(const std::vector<std::complex<Real>>& bases, const std::vector<long>& exponents, double ulps) { + using Complex = std::complex<Real>; + Index size = 2 * internal::packet_traits<Complex>::size + 1; + ArrayX<Complex> z(size), y(size); + for (long n : exponents) { + if (!has_complex_reference<Real>(n)) continue; + Exponent exponent = Exponent(n); + for (const Complex& base : bases) { + z.setConstant(base); + y = z.pow(exponent); + Complex reference = reference_complex_pow(base, n); + for (Index i = 0; i < size; ++i) { + bool ok = complex_within_ulps(y(i), reference, ulps); + if (!ok) + std::cout << "pow(" << base << ", " << n << ") = " << y(i) << " != " << reference << " (" << ulps << " ulp)" + << std::endl; + VERIFY(ok); + } + } + } +} + +template <typename Real> +void complex_pow_test() { + using Complex = std::complex<Real>; + Index size = 2 * internal::packet_traits<Complex>::size + 1; + + // Small integer inputs have exactly representable powers. + { + ArrayX<Complex> z = ArrayX<Complex>::Constant(size, Complex(3, 4)); + VERIFY((z.pow(3) == Complex(-117, 44)).all()); + VERIFY((z.pow(Real(3)) == Complex(-117, 44)).all()); + VERIFY((z.pow(4) == Complex(-527, -336)).all()); + z.setConstant(Complex(1, 1)); + VERIFY((z.pow(2) == Complex(0, 2)).all()); + VERIFY((z.pow(Real(2)) == Complex(0, 2)).all()); + VERIFY((z.pow(4) == Complex(-4, 0)).all()); + VERIFY((z.pow(-2) == Complex(0, Real(-0.5))).all()); + VERIFY((z.pow(-4) == Complex(Real(-0.25), 0)).all()); + VERIFY((z.pow(0) == Complex(1, 0)).all()); + VERIFY((z.pow(1) == Complex(1, 1)).all()); + // (1+i)^(8k+7) = 2^(4k+3) (1 - i): exact, with a magnitude of sqrt(2) times a finite component. The unscaled + // loop must not admit it once the exponent budget is exhausted, or its intermediates overflow. + constexpr int k = (std::numeric_limits<Real>::max_exponent - 8) / 4; + Real big = Real(std::ldexp(1.0, 4 * k + 3)); + VERIFY((z.pow(8 * k + 7) == Complex(big, -big)).all()); + VERIFY((z.pow(Real(8 * k + 7)) == Complex(big, -big)).all()); + } + // Components so far apart that the smaller one underflows when both take the larger one's scale, while the + // power's imaginary part, 2ab, is normal: each component is checked on its own, as the magnitude-relative check + // would not notice the imaginary part missing. + { + Real big = Real(std::ldexp(1.0, std::numeric_limits<Real>::max_exponent / 2 - 2)); + Real tiny = Real(std::ldexp(1.0, std::numeric_limits<Real>::min_exponent + std::numeric_limits<Real>::digits)); + ArrayX<Complex> z = ArrayX<Complex>::Constant(size, Complex(big, tiny)); + ArrayX<Complex> y = z.pow(2); + for (Index k = 0; k < size; ++k) { + VERIFY(within_ulps(numext::real(y(k)), big * big, 1.0)); + VERIFY(within_ulps(numext::imag(y(k)), Real(2) * big * tiny, 1.0)); + } + } + // Powers with both components subnormal, exact: 2^(5s) times (1 + i)^5 = -4 (1 + i) and (1 + i)^-5 = -(1 - i) / 8. + // Compared coefficient by coefficient, as a vectorized comparison may flush the expected value to zero too. + { + const int s = (std::numeric_limits<Real>::min_exponent - 5) / 5; + const Complex tiny = Real(std::ldexp(1.0, 5 * s + 2)) * Complex(-1, -1); + const Complex tinier = Real(std::ldexp(1.0, 5 * s - 3)) * Complex(-1, 1); + ArrayX<Complex> z = ArrayX<Complex>::Constant(size, Real(std::ldexp(1.0, s)) * Complex(1, 1)); + ArrayX<Complex> y = z.pow(5), y_real = z.pow(Real(5)); + for (Index k = 0; k < size; ++k) VERIFY(y(k) == tiny && y_real(k) == tiny); + z.setConstant(Real(std::ldexp(1.0, -s)) * Complex(1, 1)); + y = z.pow(-5); + y_real = z.pow(Real(-5)); + for (Index k = 0; k < size; ++k) VERIFY(y(k) == tinier && y_real(k) == tinier); + } + // A zero component beside nonzero ones, (1 + i)^2 = 2i and (1 + 2i)^2 = -3 + 4i in one packet, raises neither + // FE_INVALID nor FE_OVERFLOW. + { + ArrayX<Complex> z = ArrayX<Complex>::Constant(size, Complex(1, 2)), y; + z(0) = Complex(1, 1); + VERIFY_IS_EQUAL(pow_exceptions(z, 2, y), 0); + for (Index k = 0; k < size; ++k) VERIFY(y(k) == (k == 0 ? Complex(0, 2) : Complex(-3, 4))); + VERIFY_IS_EQUAL(pow_exceptions(z, Real(2), y), 0); + for (Index k = 0; k < size; ++k) VERIFY(y(k) == (k == 0 ? Complex(0, 2) : Complex(-3, 4))); + } + // Reciprocals of normal bases whose norm exceeds 1 / min, c (1 + i) with |z|^2 = 4.5 * 2^(2e), or whose smaller + // component is subnormal: 1/(2^p + 2^-30 i) = 2^-p - 2^(-30 - 2p) i to working precision. + { + const int e = (std::numeric_limits<Real>::max_exponent - 1) / 2 - 1; + const Real c = Real(std::ldexp(1.5, e)); + ArrayX<Complex> y = ArrayX<Complex>::Constant(size, Complex(c, c)).pow(-1); + for (Index k = 0; k < size; ++k) { + VERIFY(within_ulps(numext::real(y(k)), Real(1) / (Real(2) * c), 2.0)); + VERIFY(within_ulps(numext::imag(y(k)), Real(-1) / (Real(2) * c), 2.0)); + } + const int p = (std::numeric_limits<Real>::max_exponent - 1) / 2 - 13; + y = ArrayX<Complex>::Constant(size, Complex(Real(std::ldexp(1.0, p)), Real(std::ldexp(1.0, -30)))).pow(-1); + for (Index k = 0; k < size; ++k) { + VERIFY(within_ulps(numext::real(y(k)), Real(std::ldexp(1.0, -p)), 2.0)); + VERIFY(within_ulps(numext::imag(y(k)), Real(-std::ldexp(1.0, -30 - 2 * p)), 2.0)); + } + } + // Separated components near |z| = 1, with a large |n|: (a + bi)^n = a^n (1 + i n b/a) and + // (b + ai)^n = i^n a^n (1 - i n b/a) to working precision, both components to double-word accuracy. The smaller + // component is subnormal, or normal but small enough that a negative power, which shrinks between + // renormalizations, would take its residuals below the normal range. + const Complex i_powers[4] = {Complex(1, 0), Complex(0, 1), Complex(-1, 0), Complex(0, -1)}; + for (int b_exponent : {std::numeric_limits<Real>::min_exponent - 6, std::numeric_limits<Real>::min_exponent + 4}) { + Real a = Real(1) + Real(std::ldexp(1.0, -10)); + Real b = Real(std::ldexp(1.0, b_exponent)); + for (long n : {100L, 101L, 1000L, 1003L, -99L, -100L, -1000L, -1001L}) { + if (!has_complex_reference<Real>(n)) continue; + Real power = numext::real(reference_complex_pow(Complex(a), n)); + Real term = Real(n) * numext::real(reference_complex_pow(Complex(a), n - 1)) * b; + for (bool swapped : {false, true}) { + Complex expected = swapped ? i_powers[((n % 4) + 4) % 4] * Complex(power, -term) : Complex(power, term); + ArrayX<Complex> z = ArrayX<Complex>::Constant(size, swapped ? Complex(b, a) : Complex(a, b)); + ArrayX<Complex> y = z.pow(int(n)); + for (Index k = 0; k < size; ++k) { + VERIFY(within_ulps(numext::real(y(k)), numext::real(expected), 2.0)); + VERIFY(within_ulps(numext::imag(y(k)), numext::imag(expected), 2.0)); + } + } + } + } + // (1 + 2^-40 i)^n with n = 2^30 + 1, where n r = 2^-10 makes first order 8 ulp off for float: such an n + // renormalizes at every step instead. The reference is the binomial series in double, whose terms + // C(n, k) r^k fall by 2^-10 each. + EIGEN_IF_CONSTEXPR (std::is_same<Real, float>::value) { + const long n = (1L << 30) + 1; + const double r = std::ldexp(1.0, -40); + double re = 0, im = 0, term = 1; + for (int k = 0; k < 12; ++k) { + if (k > 0) term *= double(n - k + 1) / k * r; + double signed_term = (k / 2) % 2 == 0 ? term : -term; + (k % 2 == 0 ? re : im) += signed_term; + } + ArrayX<Complex> z = ArrayX<Complex>::Constant(size, Complex(1, Real(r))); + ArrayX<Complex> y = z.pow(int(n)); + for (Index k = 0; k < size; ++k) { + VERIFY(within_ulps(numext::real(y(k)), Real(re), 1.0)); + VERIFY(within_ulps(numext::imag(y(k)), Real(im), 1.0)); + } + } + // n = 2^digits + 1 is inexact in Real, and the first-order coefficient n b / a must not round it: the + // imaginary part of (1 + 1.5 * 2^e i)^(+-n) is +-(3 * 2^(digits - 1) + 1.5) 2^e, correctly rounded to + // +-(3 * 2^(digits - 1) + 2) 2^e, and the real part is 1. + { + constexpr int digits = std::numeric_limits<Real>::digits; + constexpr int e = std::numeric_limits<Real>::min_exponent - 2; + const long long n = (1LL << digits) + 1; + Real im = Real(std::ldexp(3.0 * std::ldexp(1.0, digits - 1) + 2.0, e)); + ArrayX<Complex> z = ArrayX<Complex>::Constant(size, Complex(1, Real(std::ldexp(1.5, e)))); + for (long long sign : {1LL, -1LL}) { + ArrayX<Complex> y = z.pow(sign * n); + for (Index k = 0; k < size; ++k) VERIFY(y(k) == Complex(1, Real(sign) * im)); + } + } + // An element's power does not depend on its packet neighbours, one of which here forces the scaled loop: the + // unscaled loop, whose residuals would underflow in the smaller component (here 8 a b^7, subnormal), requires + // both components in range, so this base takes the scaled loop either way. + { + constexpr int min_exponent = std::numeric_limits<Real>::min_exponent; + constexpr int e = min_exponent / 9; + for (int i = 0; i < 16; ++i) { + Real a = std::ldexp(internal::random<Real>(Real(1), Real(2)), min_exponent - 20 - 7 * e); + Real b = std::ldexp(internal::random<Real>(Real(-2), Real(-1)), e); + ArrayX<Complex> alone = ArrayX<Complex>::Constant(size, Complex(a, b)), mixed = alone; + for (Index k = 1; k < size; k += 2) mixed(k) = Complex((std::numeric_limits<Real>::max)() / 4, 1); + ArrayX<Complex> y_alone = alone.pow(8), y_mixed = mixed.pow(8); + for (Index k = 0; k < size; k += 2) VERIFY(y_alone(k) == y_mixed(k)); + } + } + // Random bases at magnitudes where the reference cannot under- or overflow; the exponent runs past the + // renormalization interval of the implementation. + { + const std::vector<long> exponents = {2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 40, -2, -3, -4, -5, -8, -9, -17, -40}; + std::vector<Complex> bases; + for (int i = 0; i < 64; ++i) { + Real magnitude = internal::random<Real>(Real(0.5), Real(2)); + Real angle = internal::random<Real>(Real(-3.14159), Real(3.14159)); + bases.push_back(Complex(magnitude * std::cos(angle), magnitude * std::sin(angle))); + } + check_complex_pow<Real, int>(bases, exponents, 2.0); + check_complex_pow<Real, Real>(bases, exponents, 2.0); + } + // Magnitudes across the range, against the same base scaled by a power of two, which only moves the exponent. + // The scale runs through the range in steps, ends included: at its extremes |z| exceeds what the reciprocal's + // norm can represent. + { + constexpr int max_exponent = std::numeric_limits<Real>::max_exponent; + for (long n : {1L, 3L, 8L, 100L, -1L, -2L, -3L, -8L, -100L}) { + int extreme = (max_exponent - 4) / int(numext::abs(n)) - 1; // |base|^n within 2^(+-|n|) stays finite + for (int i = 0; i <= 32; ++i) { + int e = -extreme + (2 * extreme * i) / 32; + Real scale = Real(std::ldexp(1.0, e)); + Real magnitude = internal::random<Real>(Real(0.5), Real(1.5)); + Real angle = internal::random<Real>(Real(-3.14159), Real(3.14159)); + Complex base(magnitude * std::cos(angle), magnitude * std::sin(angle)); + ArrayX<Complex> z = ArrayX<Complex>::Constant(size, base), w = ArrayX<Complex>::Constant(size, base * scale); + ArrayX<Complex> zn = z.pow(int(n)), wn = w.pow(int(n)); + Real power_of_scale = Real(std::pow(double(scale), double(n))); + for (Index k = 0; k < size; ++k) { + Complex expected = zn(k) * power_of_scale; + VERIFY(complex_within_ulps(wn(k), expected, 2.0)); + } + } + } + } + // Zero and infinite bases are the points 0 and infinity of the extended plane, any value with an infinite + // component being the infinity +inf + 0i of std::proj: a zero base gives +0 + 0i for n > 1 and +inf + 0i for n < 0, + // an infinite base the reverse, whatever the signs of their zeros; a NaN component without an infinite one gives + // NaN + NaN i. std::pow leaves all of these to the implementation. In a packet, in the scalar tail, and next to + // ordinary bases in the same packet, for integer and floating exponents; n = 1 returns the base. + { + Real inf = std::numeric_limits<Real>::infinity(); + Real nan = std::numeric_limits<Real>::quiet_NaN(); + auto same = [](const Complex& a, const Complex& b) { + auto same_real = [](const Real& x, const Real& y) { + return ((numext::isnan)(x) && (numext::isnan)(y)) || (x == y && std::signbit(x) == std::signbit(y)); + }; + return same_real(numext::real(a), numext::real(b)) && same_real(numext::imag(a), numext::imag(b)); + }; + std::vector<Complex> zeros = {Complex(0, 0), Complex(-Real(0), 0), Complex(0, -Real(0)), + Complex(-Real(0), -Real(0))}; + std::vector<Complex> infinities = {Complex(inf, 0), Complex(-inf, 2), Complex(0, inf), Complex(-2, -inf), + Complex(inf, inf), Complex(-inf, inf), Complex(inf, nan), Complex(nan, -inf)}; + std::vector<Complex> nans = {Complex(nan, 1), Complex(1, nan), Complex(nan, 0), Complex(nan, nan)}; + for (int n : {2, 3, 4, 5, -1, -2, -3, -4}) { + for (int kind = 0; kind < 3; ++kind) { + const std::vector<Complex>& bases = kind == 0 ? zeros : kind == 1 ? infinities : nans; + Complex expected = kind == 2 ? Complex(nan, nan) : (kind == 0) == (n > 0) ? Complex(0, 0) : Complex(inf, 0); + for (const Complex& base : bases) { + ArrayX<Complex> z = ArrayX<Complex>::Constant(size, base); + for (Index k = 1; k < size; k += 2) z(k) = Complex(Real(0.75), Real(-1.25)); + ArrayX<Complex> y = z.pow(n), y_real = z.pow(Real(n)), ordinary = ArrayX<Complex>::Constant(1, z(1)).pow(n); + for (Index k = 0; k < size; k += 2) VERIFY(same(y(k), expected) && same(y_real(k), expected)); + for (Index k = 1; k < size; k += 2) VERIFY(same(y(k), ordinary(0))); + VERIFY(same(ArrayX<Complex>::Constant(1, base).pow(1)(0), base)); + } + } + } + } + // A base with one zero component is the limit of a base whose component is tiny and of the same sign, and the + // zero component of its power takes the sign of that limit: here of the power of the perturbed base, computed + // in double by repeated multiplication, where the tiny component cannot cancel. + for (Real a : {Real(2), Real(-2), Real(0.5), Real(-3)}) { + for (Real s : {Real(0), -Real(0)}) { + for (bool swapped : {false, true}) { + for (int n = -12; n <= 12; ++n) { + if (n == 0) continue; + double tiny = std::copysign(std::ldexp(1.0, -60), double(s)); + std::complex<double> w = swapped ? std::complex<double>(tiny, a) : std::complex<double>(a, tiny); + if (n < 0) w = std::conj(w) / std::norm(w); + std::complex<double> reference = w; + for (int k = 1; k < numext::abs(n); ++k) reference *= w; + bool real_is_zero = std::abs(reference.real()) < std::abs(reference.imag()); + double zero_sign = real_is_zero ? reference.real() : reference.imag(); + ArrayX<Complex> z = ArrayX<Complex>::Constant(size, swapped ? Complex(s, a) : Complex(a, s)); + ArrayX<Complex> y = z.pow(n); + for (Index k = 0; k < size; ++k) { + Real zero = real_is_zero ? numext::real(y(k)) : numext::imag(y(k)); + VERIFY(zero == Real(0) && std::signbit(zero) == std::signbit(zero_sign)); + } + } + } + } + } +} + +// half and bfloat16 keep plain repeated squaring, whose error grows like n ulp, for integer exponents and small +// floating-point ones, and generic_pow (pow through float) beyond; with F16C their packets take the vectorized +// path. Check both against float. +template <typename Scalar> +void narrow_pow_test() { + Index size = 2 * internal::packet_traits<Scalar>::size + 1; + ArrayX<Scalar> x(size), y(size); + for (long n : {1L, 2L, 3L, 5L, 7L, 8L, 16L, 100L, 4097L, -1L, -2L, -3L, -8L, -100L}) { + // Powers within the normal range of the type: |x|^n in 2^(+-(max_exponent - 2)). + double limit = double(std::numeric_limits<Scalar>::max_exponent - 2) / double(numext::abs(n)); + for (int i = 0; i < 32; ++i) { + float base = float(std::exp2(internal::random<double>(-limit, limit))); + x.setConstant(Scalar(i % 2 ? -base : base)); + Scalar reference = Scalar(std::pow(float(x(0)), float(n))); + double ulps = 2.0 + double(numext::abs(n)); + y = x.pow(int(n)); + for (Index k = 0; k < size; ++k) { + if (!within_ulps(y(k), reference, ulps)) + std::cout << "pow(" << x(0) << ", " << n << ") = " << y(k) << " != " << reference << std::endl; + VERIFY(within_ulps(y(k), reference, ulps)); + } + if (n == long(Scalar(float(n)))) { + y = x.pow(Scalar(float(n))); + for (Index k = 0; k < size; ++k) VERIFY(within_ulps(y(k), reference, ulps)); + } + } + } +} + +// complex<long double> keeps plain squaring; its reciprocal must stay range-safe, as the double-word path's +// conj(z)/|z|^2 would not be for long double's wider exponent range. +void complex_long_double_range_test() { + using Complex = std::complex<long double>; + constexpr int max_exponent = std::numeric_limits<long double>::max_exponent; + if (max_exponent <= std::numeric_limits<double>::max_exponent) return; + ArrayX<Complex> z(9), y(9); + auto close = [](const Complex& a, const Complex& b, long double ulps) { + return std::abs(a - b) <= ulps * std::numeric_limits<long double>::epsilon() * std::abs(b); + }; + // Built with ldexp rather than written as literals: 1e3000L and the like are out of range, and so a + // compile-time error, wherever long double is binary64, which the run-time guard above cannot prevent. + for (int e : {(4 * max_exponent) / 5, -(4 * max_exponent) / 5, (2 * max_exponent) / 5, -(2 * max_exponent) / 5}) { + long double magnitude = std::ldexp(1.0L, e); + z.setConstant(Complex(magnitude, magnitude / 3)); + y = z.pow(-1); + Complex expected = Complex(1) / z(0); + for (Index k = 0; k < 9; ++k) VERIFY(close(y(k), expected, 4.0L)); + if (2 * numext::abs(e) >= max_exponent) continue; // the square would leave the range + y = z.pow(-2); + expected = Complex(1) / (z(0) * z(0)); + for (Index k = 0; k < 9; ++k) VERIFY(close(y(k), expected, 8.0L)); + } +} + +EIGEN_DECLARE_TEST(unary_pow) { + for (int i = 0; i < g_repeat; i++) { + CALL_SUBTEST_1(real_pow_test<float>()); + CALL_SUBTEST_2(real_pow_test<double>()); + CALL_SUBTEST_3(complex_pow_test<float>()); + CALL_SUBTEST_4(complex_pow_test<double>()); + CALL_SUBTEST_5(narrow_pow_test<half>()); + CALL_SUBTEST_6(narrow_pow_test<bfloat16>()); + CALL_SUBTEST_7(complex_long_double_range_test()); + } +}
diff --git a/test/vector_sme.cpp b/test/vector_sme.cpp new file mode 100644 index 0000000..8dc638f --- /dev/null +++ b/test/vector_sme.cpp
@@ -0,0 +1,499 @@ +// SPDX-FileCopyrightText: The Eigen Authors +// SPDX-License-Identifier: MPL-2.0 +#include "main.h" +#include <Eigen/Core> +#include <cfenv> + +#ifndef EIGEN_VECTORIZE_SME +#error "vector_sme requires an SME2 target" +#endif + +template <typename Lhs, typename Rhs> +void sme_check_dot(const Lhs& lhs, const Rhs& rhs) { + using Scalar = typename Lhs::Scalar; + long double expected = 0, magnitude = 0; + for (Index i = 0; i < lhs.size(); ++i) { + const long double term = static_cast<long double>(lhs[i]) * static_cast<long double>(rhs[i]); + expected += term; + magnitude += numext::abs(term); + } + // gamma_(2*n+4) bounds multiplication and accumulation in both the kernel and reference. + const long double rounding = + (2 * static_cast<long double>(lhs.size()) + 4) * static_cast<long double>(NumTraits<Scalar>::epsilon()); + const long double bound = rounding / (1 - rounding) * magnitude; + const Scalar actual = lhs.dot(rhs); + VERIFY(rounding < 1 && (numext::isfinite)(bound)); + VERIFY((numext::isfinite)(actual)); + VERIFY(numext::abs(static_cast<long double>(actual) - expected) <= bound); +} + +template <typename Scalar> +void sme_vectors() { + using Vec = Vector<Scalar, Dynamic>; + using Row = Matrix<Scalar, 1, Dynamic>; + using Strided = Map<Vec, 0, InnerStride<Dynamic>>; + STATIC_CHECK((internal::sme_dot_supported<Vec, Row>::value)); + STATIC_CHECK((internal::sme_dot_supported<Strided, Vec>::value)); + STATIC_CHECK((!internal::sme_dot_supported<decltype(std::declval<Vec>() + std::declval<Vec>()), Vec>::value)); + STATIC_CHECK((!internal::sme_vector_scalar<std::complex<Scalar>>::value)); + STATIC_CHECK((!internal::sme_vector_scalar<int>::value)); + + std::vector<Index> sizes; + for (Index n = 0; n <= 260; ++n) { + sizes.push_back(n); + sizes.push_back(32768 + n); + } + for (Index n : {511, 512, 513, 1023, 1024, 1025, 2047, 2048, 2049, 4095, 4096, + 4097, 8191, 8192, 8193, 16383, 16384, 16385, 32767, 32768, 32769}) + sizes.push_back(n); + for (Index n : sizes) { + Vec x = Vec::Ones(n), storage = Vec::Ones(n + 2); + VERIFY_IS_EQUAL(x.dot(storage.head(n)), Scalar(n)); + x.setRandom(); + storage.setRandom(); + const Vec before = storage; + auto y = storage.segment(1, n); + sme_check_dot(x, y); + sme_check_dot(x.transpose(), y); + for (int side = 0; side < 2; ++side) { + storage = before; + if (side == 0) + y += Scalar(0.75) * x; + else + y += x * Scalar(0.75); + for (Index i = 0; i < n; ++i) { + const long double expected = static_cast<long double>(before[i + 1]) + 0.75L * static_cast<long double>(x[i]); + const long double magnitude = + numext::abs(static_cast<long double>(before[i + 1])) + 0.75L * numext::abs(static_cast<long double>(x[i])); + VERIFY(numext::abs(static_cast<long double>(y[i]) - expected) <= + 4 * static_cast<long double>(NumTraits<Scalar>::epsilon()) * magnitude); + } + VERIFY_IS_EQUAL(storage[0], before[0]); + VERIFY_IS_EQUAL(storage[n + 1], before[n + 1]); + } + Vec alias = x; + alias += Scalar(0.75) * alias; + for (Index i = 0; i < n; ++i) { + const long double expected = static_cast<long double>(x[i]) * 1.75L; + VERIFY(numext::abs(static_cast<long double>(alias[i]) - expected) <= + 4 * static_cast<long double>(NumTraits<Scalar>::epsilon()) * numext::abs(expected)); + } + Strided contiguous(x.data(), n, InnerStride<Dynamic>(1)); + sme_check_dot(contiguous, y); + } + Vec x = Vec::Random(65538), y = Vec::Random(32769), expected = x; + Strided strided(x.data(), y.size(), InnerStride<Dynamic>(2)); + Strided negative(x.data() + x.size() - 2, y.size(), InnerStride<Dynamic>(-2)); + sme_check_dot(strided, y); + sme_check_dot(negative, y); + for (Index i = 0; i < y.size(); ++i) expected[2 * i] += Scalar(0.75) * y[i]; + strided += Scalar(0.75) * y; + VERIFY_IS_APPROX(x, expected); + sme_check_dot(-y, strided); + sme_check_dot(y + y, strided); + sme_check_dot(y.reverse(), strided); + VERIFY_RAISES_ASSERT(y.dot(x)); + + for (int pattern = 0; pattern < 3; ++pattern) { + Vec a = Vec::Random(65537), b = Vec::Random(65537); + for (Index i = 0; i < a.size(); ++i) { + if (pattern == 0) b[i] = (i & 1) ? -a[i] : a[i]; + if (pattern == 1) a[i] = numext::ldexp(a[i], int(i % 81) - 40); + if (pattern == 2) { + a[i] = Scalar(1); + b[i] = i % 3 == 0 ? Scalar(1) / NumTraits<Scalar>::epsilon() + : i % 3 == 1 ? Scalar(1) + : -Scalar(1) / NumTraits<Scalar>::epsilon(); + } + } + sme_check_dot(a, b); + } + for (Index n : {524287, 524288, 524289, 1048575, 1048576, 1048577}) { + Vec a = Vec::Ones(n), b = Vec::Ones(n); + VERIFY_IS_EQUAL(a.dot(b), Scalar(n)); + b += Scalar(0.75) * a; + VERIFY((b.array() == Scalar(1.75)).all()); + } +} + +template <typename Scalar> +void sme_vector_cache_sizes() { + using Vec = Vector<Scalar, Dynamic>; + const std::ptrdiff_t old_l1 = l1CacheSize(), old_l2 = l2CacheSize(), old_l3 = l3CacheSize(); + const Index n = 32769; + const std::ptrdiff_t bytes = n * sizeof(Scalar); + const Vec x = Vec::Ones(n); + Vec y(n); + for (std::ptrdiff_t l2 : {bytes, bytes - 1, std::ptrdiff_t(0), std::ptrdiff_t(-1), bytes + 1}) { + setCpuCacheSizes(old_l1, l2, old_l3); + VERIFY_IS_EQUAL(internal::sme_vector_size_suitable<Scalar>(n), l2 >= bytes); + VERIFY(!internal::sme_vector_size_suitable<Scalar>((std::numeric_limits<Index>::max)())); + VERIFY_IS_EQUAL(x.dot(x), Scalar(n)); + y.setOnes(); + y += Scalar(0.75) * x; + VERIFY((y.array() == Scalar(1.75)).all()); + y.setOnes(); + y += x * Scalar(0.75); + VERIFY((y.array() == Scalar(1.75)).all()); + } + for (std::ptrdiff_t l1 : {bytes, bytes - 1, std::ptrdiff_t(0), std::ptrdiff_t(-1), bytes + 1}) { + setCpuCacheSizes(l1, bytes, old_l3); + const bool suitable = l1 > 0 && l1 <= bytes; + VERIFY_IS_EQUAL(internal::sme_vector_size_suitable<Scalar>(n), suitable); + y.setOnes(); + VERIFY_IS_EQUAL(internal::sme_try_axpy(y, x, Scalar(0.75)), suitable); + VERIFY((y.array() == (suitable ? Scalar(1.75) : Scalar(1))).all()); + VERIFY_IS_EQUAL(x.dot(x), Scalar(n)); + } + for (std::ptrdiff_t l1 : {2 * bytes - 1, 2 * bytes, 2 * bytes + 1}) { + setCpuCacheSizes(l1, bytes, old_l3); + VERIFY_IS_EQUAL((internal::sme_vector_size_suitable<Scalar, 2>(n)), l1 <= 2 * bytes); + VERIFY_IS_EQUAL(x.dot(x), Scalar(n)); + } +#ifndef EIGEN_USE_BLAS + const std::ptrdiff_t matrix_bytes = 128 * 4 * sizeof(Scalar); + for (std::ptrdiff_t l1 : {matrix_bytes - 1, matrix_bytes, matrix_bytes + 1, std::ptrdiff_t(0)}) { + setCpuCacheSizes(l1, old_l2, old_l3); + VERIFY_IS_EQUAL(internal::sme_gemv_size_suitable<Scalar>(Index(128), Index(4)), l1 > 0 && l1 <= matrix_bytes); + VERIFY(!internal::sme_gemv_size_suitable<Scalar>(Index(0), Index(4))); + } +#endif + setCpuCacheSizes(old_l1, old_l2, old_l3); +} + +template <typename Scalar> +void sme_vector_scalar_factors() { + using Vec = Vector<Scalar, Dynamic>; + const Index n = 32769; + Vec x = Vec::Constant(n, Scalar(0.25)), y = Vec::Zero(n); + y += x.cwiseProduct(x); + VERIFY((y.array() == Scalar(0.0625)).all()); + y.setZero(); + y += Vec::Constant(n, Scalar(0.75)).cwiseProduct(x); + VERIFY((y.array() == Scalar(0.1875)).all()); + + // Folding the two scale factors would overflow even though every output is finite. + const Scalar large = (std::numeric_limits<Scalar>::max)() / Scalar(2); + y.setZero(); + y += large * (Scalar(4) * x); + VERIFY((y.array() == large).all()); + y.setZero(); + y += (x * Scalar(4)) * large; + VERIFY((y.array() == large).all()); + + Array<Scalar, Dynamic, 1> a = x.array(), b = Array<Scalar, Dynamic, 1>::Zero(n); + b += Scalar(0.75) * a; + VERIFY((b == Scalar(0.1875)).all()); + b.setZero(); + b += a * Scalar(0.75); + VERIFY((b == Scalar(0.1875)).all()); +} + +template <typename Scalar> +void sme_matrix_vectors() { + using Vec = Vector<Scalar, Dynamic>; + using Mat = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; + for (Index rows : {127, 128, 129, 255, 256, 257, 513}) { + for (Index cols : {3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65, 129}) { + for (Index padding : {0, 3}) { + Mat storage = Mat::Random(rows + padding, cols); + auto a = storage.topRows(rows); + Vec x = Vec::Random(cols), original = Vec::Random(rows), result(rows); + for (Scalar alpha : {Scalar(0), Scalar(0.75), Scalar(-1)}) { + result = original; + result.noalias() += alpha * a * x; + for (Index i = 0; i < rows; ++i) { + long double expected = static_cast<long double>(original[i]), magnitude = numext::abs(expected); + for (Index j = 0; j < cols; ++j) { + const long double term = + static_cast<long double>(alpha) * static_cast<long double>(a(i, j)) * static_cast<long double>(x[j]); + expected += term; + magnitude += numext::abs(term); + } + const long double bound = + 8 * (cols + 1) * static_cast<long double>(NumTraits<Scalar>::epsilon()) * magnitude; + VERIFY((numext::isfinite)(bound)); + VERIFY(numext::abs(static_cast<long double>(result[i]) - expected) <= bound); + } + } + Vec strided_storage = Vec::Random(2 * rows), untouched = strided_storage; + Map<Vec, 0, InnerStride<2>> strided(strided_storage.data(), rows); + Vec reference = strided; + reference.noalias() += a * x; + strided.noalias() += a * x; + VERIFY_IS_APPROX(strided, reference); + for (Index i = 0; i < rows; ++i) VERIFY_IS_EQUAL(strided_storage[2 * i + 1], untouched[2 * i + 1]); + } + } + } +} + +template <typename Scalar> +void sme_vector_alignment() { + using Vec = Vector<Scalar, Dynamic>; + const Index padding = 64 / sizeof(Scalar); + for (Index tail : {0, 1, 15, 31, 63, 127, 255, 256}) { + const Index n = 32768 + tail; + Vec x_storage = Vec::Ones(n + padding), y_storage(n + 2 * padding); + for (Index offset = 0; offset < padding; ++offset) { + y_storage.setConstant(Scalar(2)); + const auto x = x_storage.segment(offset, n); + auto y = y_storage.segment(padding + offset, n); + VERIFY(internal::sme_try_axpy(y, x, Scalar(0.75))); + VERIFY((y.array() == Scalar(2.75)).all()); + VERIFY((y_storage.head(padding + offset).array() == Scalar(2)).all()); + VERIFY((y_storage.tail(padding - offset).array() == Scalar(2)).all()); + VERIFY_IS_EQUAL(x.dot(y), Scalar(2.75) * Scalar(n)); + } + } +} + +template <typename Scalar> +void sme_gemv_tails() { + using Vec = Vector<Scalar, Dynamic>; + using Mat = Matrix<Scalar, Dynamic, Dynamic>; + // Every predicated chunk pattern of a row block, including at the largest architectural SVL. + for (Index rows = 128; rows <= 1100; ++rows) { + const Mat a = Mat::Ones(rows, 65); + const Vec x = Vec::Ones(65); + Vec storage = Vec::Ones(rows + 2); + auto y = storage.segment(1, rows); + y.noalias() += Scalar(0.75) * a * x; + VERIFY((y.array() == Scalar(49.75)).all()); + VERIFY_IS_EQUAL(storage[0], Scalar(1)); + VERIFY_IS_EQUAL(storage[rows + 1], Scalar(1)); + } +} + +template <typename Scalar> +void sme_vector_special_values() { + using Vec = Vector<Scalar, Dynamic>; + using Mat = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; + const Scalar inf = NumTraits<Scalar>::infinity(), nan = NumTraits<Scalar>::quiet_NaN(); + const Scalar tiny = (std::numeric_limits<Scalar>::denorm_min)(); + for (Index pos : {0, 7, 31, 63, 127, 255, 256, 32768}) { + Vec x = Vec::Zero(32769), y = Vec::Ones(32769), result(32769); + for (Scalar special : {inf, -inf, nan, tiny, -tiny, Scalar(-0.0)}) { + x[pos] = special; + const Scalar actual = x.dot(y); + if ((numext::isnan)(special)) + VERIFY((numext::isnan)(actual)); + else + VERIFY_IS_EQUAL(actual, special); + result.setZero(); + result += Scalar(1) * x; + for (Index i = 0; i < result.size(); ++i) { + const Scalar expected = x[i] + Scalar(0); + if ((numext::isnan)(expected)) + VERIFY((numext::isnan)(result[i])); + else { + VERIFY_IS_EQUAL(result[i], expected); + VERIFY_IS_EQUAL(std::signbit(result[i]), std::signbit(expected)); + } + } + } + } + for (Scalar special : {inf, -inf, nan, tiny, -tiny}) { + Mat a = Mat::Zero(257, 65); + Vec x = Vec::Ones(65), y = Vec::Zero(257); + a(0, 0) = a(256, 64) = special; + y.noalias() += a * x; + for (Index i = 0; i < y.size(); ++i) { + const Scalar expected = i == 0 || i == 256 ? special : Scalar(0); + if ((numext::isnan)(expected)) + VERIFY((numext::isnan)(y[i])); + else + VERIFY_IS_EQUAL(y[i], expected); + } + y.setOnes(); + y.noalias() += Scalar(0) * a * x; + VERIFY((y.array() == Scalar(1)).all()); + } + Vec x = Vec::Zero(32769), y = Vec::Zero(32769); + x[0] = inf; + VERIFY((numext::isnan)(x.dot(y))); +} + +template <typename Scalar> +void sme_gemv_scaled_range() { + using Mat = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; + using Vec = Vector<Scalar, Dynamic>; + const Scalar largest = (std::numeric_limits<Scalar>::max)(); + const std::ptrdiff_t l1 = l1CacheSize(), l2 = l2CacheSize(), l3 = l3CacheSize(); + for (std::ptrdiff_t cache : {l1, std::ptrdiff_t(0)}) { + setCpuCacheSizes(cache, l2, l3); + for (Index rows : {256, 257}) { + for (Index cols : {128, 129}) { + Mat matrix = Mat::Constant(rows, cols, largest / Scalar(64)); + Vec rhs = Vec::Ones(cols), result = Vec::Zero(rows); + result.noalias() += Scalar(0.25) * matrix * rhs; + const Scalar expected = (largest / Scalar(256)) * Scalar(cols); + const Scalar bound = Scalar(4 * cols) * NumTraits<Scalar>::epsilon(); + for (Index i = 0; i < rows; ++i) { + VERIFY((numext::isfinite)(result[i])); + VERIFY(numext::abs(result[i] / expected - Scalar(1)) <= bound); + } + } + } + } + setCpuCacheSizes(l1, l2, l3); +} + +template <typename Scalar> +void sme_dot_signed_zero() { + using Vec = Vector<Scalar, Dynamic>; + const Scalar tiny = (std::numeric_limits<Scalar>::denorm_min)(); + for (Index n : {32768, 32769, 32799, 33791}) { + Vec x = Vec::Constant(n, Scalar(-0.0)), y = Vec::Ones(n); + const Scalar negative = x.dot(y); + VERIFY_IS_EQUAL(negative, Scalar(0)); + VERIFY((numext::signbit)(negative)); + x.setZero(); + const Scalar positive = x.dot(y); + VERIFY_IS_EQUAL(positive, Scalar(0)); + VERIFY(!(numext::signbit)(positive)); + // A single positive-zero product, in the first group or the last element, gives +0. + for (Index pos : {Index(0), n - 1}) { + x.setConstant(Scalar(-0.0)); + x[pos] = Scalar(0); + const Scalar mixed = x.dot(y); + VERIFY_IS_EQUAL(mixed, Scalar(0)); + VERIFY(!(numext::signbit)(mixed)); + } + // Exact cancellation, across lanes (adjacent terms) or within a fused lane (halves), rounds to +0. + for (int within_lane = 0; within_lane < 2; ++within_lane) { + x.setConstant(Scalar(-0.0)); + const Index half = n / 2; + for (Index i = 0; i < half; ++i) { + x[within_lane ? i : 2 * i] = Scalar(1); + x[within_lane ? half + i : 2 * i + 1] = Scalar(-1); + } + const Scalar cancelled = x.dot(y); + VERIFY_IS_EQUAL(cancelled, Scalar(0)); + VERIFY(!(numext::signbit)(cancelled)); + } + // Products that underflow to -0 keep the sign of every fused partial sum. + x.setConstant(-tiny); + const Scalar underflow = x.dot(Vec::Constant(n, Scalar(0.25))); + VERIFY_IS_EQUAL(underflow, Scalar(0)); + VERIFY((numext::signbit)(underflow)); + } +} + +template <typename Scalar> +void sme_dot_directed_rounding() { +#if EIGEN_COMP_CLANG +#pragma STDC FENV_ACCESS ON +#endif + using Vec = Vector<Scalar, Dynamic>; + const int saved = std::fegetround(); + const Index n = 32769; + const Vec y = Vec::Ones(n); + for (int mode : {FE_TONEAREST, FE_UPWARD, FE_DOWNWARD, FE_TOWARDZERO}) { + VERIFY_IS_EQUAL(std::fesetround(mode), 0); + // IEEE 754 6.3: x + x keeps the sign of x; an exact zero sum of opposite signs is -0 only when rounding down. + Vec x = Vec::Zero(n); + const Scalar positive = x.dot(y); + x.setConstant(Scalar(-0.0)); + const Scalar negative = x.dot(y); + x[0] = Scalar(0); + const Scalar mixed = x.dot(y); + std::fesetround(saved); + VERIFY(positive == Scalar(0) && !std::signbit(positive)); + VERIFY(negative == Scalar(0) && std::signbit(negative)); + VERIFY(mixed == Scalar(0) && std::signbit(mixed) == (mode == FE_DOWNWARD)); + } +} + +template <typename Scalar> +void sme_vector_exception_state() { +#if EIGEN_COMP_CLANG +#pragma STDC FENV_ACCESS ON +#endif + using Vec = Vector<Scalar, Dynamic>; + using Mat = Matrix<Scalar, Dynamic, Dynamic, ColMajor>; + std::fenv_t saved; + std::fegetenv(&saved); + const Scalar inf = NumTraits<Scalar>::infinity(), largest = (std::numeric_limits<Scalar>::max)(); + // The kernels leave the caller's flags as they were, whether or not an SVE tail or row remainder follows the + // ZA loop: exact inputs, then inf * 0 (invalid), then max * 2 (overflow). + for (Index n : {32768, 32769}) { + Vec x(n), y(n); + Mat matrix(n == 32768 ? 256 : 257, 128); + Vec rhs(128), result(matrix.rows()); + for (int input = 0; input < 3; ++input) { + const Scalar value = input == 0 ? Scalar(1) : input == 1 ? inf : largest; + const Scalar factor = input == 0 ? Scalar(1) : input == 1 ? Scalar(0) : Scalar(2); + for (int prior : {0, FE_DIVBYZERO}) { + for (int operation = 0; operation < 3; ++operation) { + x.setConstant(value); + y.setConstant(factor); + matrix.setConstant(value); + rhs.setConstant(factor); + result.setZero(); + std::feclearexcept(FE_ALL_EXCEPT); + std::feraiseexcept(prior); + Scalar actual; + if (operation == 0) { + actual = x.dot(y); + } else if (operation == 1) { + y += factor * x; + actual = y[n - 1]; + } else { + result.noalias() += matrix * rhs; + actual = result[result.size() - 1]; + } + const int flags = std::fetestexcept(FE_ALL_EXCEPT); + VERIFY_IS_EQUAL(flags, prior); + if (input == 1) { + VERIFY((numext::isnan)(actual)); + } else if (input == 2) { + VERIFY_IS_EQUAL(actual, inf); + } else { + VERIFY_IS_EQUAL(actual, operation == 0 ? Scalar(n) : operation == 1 ? Scalar(2) : Scalar(128)); + } + } + } + } + } + // Longer calls are more likely to be preempted while streaming. + Vec x = Vec::Ones(Index(1) << 20), y = Vec::Ones(Index(1) << 20); + for (int repetition = 0; repetition < 64; ++repetition) { + std::feclearexcept(FE_ALL_EXCEPT); + const Scalar dot = x.dot(y); + y += Scalar(repetition % 2 ? -0.75 : 0.75) * x; + VERIFY_IS_EQUAL(std::fetestexcept(FE_ALL_EXCEPT), 0); + VERIFY_IS_EQUAL(dot, Scalar(repetition % 2 ? 1.75 : 1) * Scalar(x.size())); + } + std::fesetenv(&saved); +} + +EIGEN_DECLARE_TEST(vector_sme) { + const std::ptrdiff_t l1 = l1CacheSize(), l2 = l2CacheSize(), l3 = l3CacheSize(); + setCpuCacheSizes(128 * 1024, 4 * 1024 * 1024, l3); + CALL_SUBTEST_1(sme_vector_cache_sizes<float>()); + CALL_SUBTEST_1(sme_vector_scalar_factors<float>()); + CALL_SUBTEST_1(sme_vectors<float>()); + CALL_SUBTEST_1(sme_matrix_vectors<float>()); + CALL_SUBTEST_1(sme_vector_alignment<float>()); + CALL_SUBTEST_1(sme_gemv_tails<float>()); + CALL_SUBTEST_1(sme_vector_special_values<float>()); + CALL_SUBTEST_3(sme_gemv_scaled_range<float>()); + CALL_SUBTEST_3(sme_dot_signed_zero<float>()); + CALL_SUBTEST_3(sme_dot_directed_rounding<float>()); + CALL_SUBTEST_3(sme_vector_exception_state<float>()); +#ifdef EIGEN_VECTORIZE_SME_F64F64 + CALL_SUBTEST_2(sme_vector_cache_sizes<double>()); + CALL_SUBTEST_2(sme_vector_scalar_factors<double>()); + CALL_SUBTEST_2(sme_vectors<double>()); + CALL_SUBTEST_2(sme_matrix_vectors<double>()); + CALL_SUBTEST_2(sme_vector_alignment<double>()); + CALL_SUBTEST_2(sme_gemv_tails<double>()); + CALL_SUBTEST_2(sme_vector_special_values<double>()); + CALL_SUBTEST_4(sme_gemv_scaled_range<double>()); + CALL_SUBTEST_4(sme_dot_signed_zero<double>()); + CALL_SUBTEST_4(sme_dot_directed_rounding<double>()); + CALL_SUBTEST_4(sme_vector_exception_state<double>()); +#endif + setCpuCacheSizes(l1, l2, l3); +}
diff --git a/test/visitor.cpp b/test/visitor.cpp index ce49aad..66d3ed9 100644 --- a/test/visitor.cpp +++ b/test/visitor.cpp
@@ -560,6 +560,81 @@ } } +// A vectorizable scalar whose packet type has no comparison ops, as in cppduals (#3195). +struct NoPacketCmpScalar { + double v; + NoPacketCmpScalar(double x = 0) : v(x) {} + bool operator<(const NoPacketCmpScalar& b) const { return v < b.v; } + bool operator>(const NoPacketCmpScalar& b) const { return v > b.v; } + bool operator==(const NoPacketCmpScalar& b) const { return v == b.v; } + bool operator!=(const NoPacketCmpScalar& b) const { return v != b.v; } +}; +struct NoPacketCmpPacket { + NoPacketCmpScalar v[2]; +}; + +namespace Eigen { +template <> +struct NumTraits<NoPacketCmpScalar> : GenericNumTraits<NoPacketCmpScalar> {}; +namespace internal { +template <> +struct packet_traits<NoPacketCmpScalar> : default_packet_traits { + using type = NoPacketCmpPacket; + using half = NoPacketCmpPacket; + enum { Vectorizable = 1, AlignedOnScalar = 0, size = 2, HasCmp = 0, HasMin = 0, HasMax = 0 }; +}; +template <> +struct unpacket_traits<NoPacketCmpPacket> : default_unpacket_traits { + using type = NoPacketCmpScalar; + using half = NoPacketCmpPacket; + enum { size = 2, alignment = Unaligned, vectorizable = true }; +}; +template <> +NoPacketCmpPacket pload<NoPacketCmpPacket>(const NoPacketCmpScalar* from) { + return NoPacketCmpPacket{{from[0], from[1]}}; +} +template <> +NoPacketCmpPacket ploadu<NoPacketCmpPacket>(const NoPacketCmpScalar* from) { + return NoPacketCmpPacket{{from[0], from[1]}}; +} +template <> +NoPacketCmpPacket pset1<NoPacketCmpPacket>(const NoPacketCmpScalar& a) { + return NoPacketCmpPacket{{a, a}}; +} +template <> +void pstore<NoPacketCmpScalar, NoPacketCmpPacket>(NoPacketCmpScalar* to, const NoPacketCmpPacket& from) { + to[0] = from.v[0]; + to[1] = from.v[1]; +} +template <> +void pstoreu<NoPacketCmpScalar, NoPacketCmpPacket>(NoPacketCmpScalar* to, const NoPacketCmpPacket& from) { + to[0] = from.v[0]; + to[1] = from.v[1]; +} +} // namespace internal +} // namespace Eigen + +template <int Options> +void checkFindCoeffWithoutPacketCmp() { + using S = NoPacketCmpScalar; + Matrix<S, Dynamic, Dynamic, Options> m(5, 7); + for (Index k = 0; k < m.size(); ++k) m(k) = S(double((k * 11) % m.size())); + const S lo(-1), hi(100); + m(3, 2) = lo; + m(1, 4) = hi; + Index row, col; + VERIFY(m.minCoeff(&row, &col) == lo && row == 3 && col == 2); + VERIFY(m.maxCoeff(&row, &col) == hi && row == 1 && col == 4); + VERIFY(m.template minCoeff<PropagateNaN>(&row, &col) == lo && row == 3 && col == 2); + VERIFY(m.template maxCoeff<PropagateNaN>(&row, &col) == hi && row == 1 && col == 4); + VERIFY(m.template minCoeff<PropagateNumbers>(&row, &col) == lo && row == 3 && col == 2); + VERIFY(m.template maxCoeff<PropagateNumbers>(&row, &col) == hi && row == 1 && col == 4); + Matrix<S, Dynamic, 1> v = m.col(4); + Index index; + VERIFY(v.maxCoeff(&index) == hi && index == 1); + VERIFY(v.template minCoeff<PropagateNaN>(&index) == v.minCoeff()); +} + EIGEN_DECLARE_TEST(visitor) { for (int i = 0; i < g_repeat; i++) { CALL_SUBTEST_1(matrixVisitor(Matrix<float, 1, 1>())); @@ -595,4 +670,6 @@ CALL_SUBTEST_14(checkVisitorShortCircuit<RowMajor>()); CALL_SUBTEST_14((checkVisitorShortCircuit<ColMajor, true>())); CALL_SUBTEST_14((checkVisitorShortCircuit<RowMajor, true>())); + CALL_SUBTEST_15(checkFindCoeffWithoutPacketCmp<ColMajor>()); + CALL_SUBTEST_15(checkFindCoeffWithoutPacketCmp<RowMajor>()); }