1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2007-2011 Benoit Jacob <jacob.benoit.1@gmail.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10
11#ifndef EIGEN_CORE_H
12#define EIGEN_CORE_H
13
14// first thing Eigen does: stop the compiler from committing suicide
15#include "src/Core/util/DisableStupidWarnings.h"
16
17#if defined(__CUDACC__) && !defined(EIGEN_NO_CUDA)
18 #define EIGEN_CUDACC __CUDACC__
19#endif
20
21#if defined(__CUDA_ARCH__) && !defined(EIGEN_NO_CUDA)
22 #define EIGEN_CUDA_ARCH __CUDA_ARCH__
23#endif
24
25#if defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 9)
26#define EIGEN_CUDACC_VER ((__CUDACC_VER_MAJOR__ * 10000) + (__CUDACC_VER_MINOR__ * 100))
27#elif defined(__CUDACC_VER__)
28#define EIGEN_CUDACC_VER __CUDACC_VER__
29#else
30#define EIGEN_CUDACC_VER 0
31#endif
32
33// Handle NVCC/CUDA/SYCL
34#if defined(__CUDACC__) || defined(__SYCL_DEVICE_ONLY__)
35 // Do not try asserts on CUDA and SYCL!
36 #ifndef EIGEN_NO_DEBUG
37 #define EIGEN_NO_DEBUG
38 #endif
39
40 #ifdef EIGEN_INTERNAL_DEBUGGING
41 #undef EIGEN_INTERNAL_DEBUGGING
42 #endif
43
44 #ifdef EIGEN_EXCEPTIONS
45 #undef EIGEN_EXCEPTIONS
46 #endif
47
48 // All functions callable from CUDA code must be qualified with __device__
49 #ifdef __CUDACC__
50 // Do not try to vectorize on CUDA and SYCL!
51 #ifndef EIGEN_DONT_VECTORIZE
52 #define EIGEN_DONT_VECTORIZE
53 #endif
54
55 #define EIGEN_DEVICE_FUNC __host__ __device__
56 // We need cuda_runtime.h to ensure that that EIGEN_USING_STD_MATH macro
57 // works properly on the device side
58 #include <cuda_runtime.h>
59 #else
60 #define EIGEN_DEVICE_FUNC
61 #endif
62
63#else
64 #define EIGEN_DEVICE_FUNC
65
66#endif
67
68// When compiling CUDA device code with NVCC, pull in math functions from the
69// global namespace. In host mode, and when device doee with clang, use the
70// std versions.
71#if defined(__CUDA_ARCH__) && defined(__NVCC__)
72 #define EIGEN_USING_STD_MATH(FUNC) using ::FUNC;
73#else
74 #define EIGEN_USING_STD_MATH(FUNC) using std::FUNC;
75#endif
76
77#if (defined(_CPPUNWIND) || defined(__EXCEPTIONS)) && !defined(__CUDA_ARCH__) && !defined(EIGEN_EXCEPTIONS) && !defined(EIGEN_USE_SYCL)
78 #define EIGEN_EXCEPTIONS
79#endif
80
81#ifdef EIGEN_EXCEPTIONS
82 #include <new>
83#endif
84
85// then include this file where all our macros are defined. It's really important to do it first because
86// it's where we do all the alignment settings (platform detection and honoring the user's will if he
87// defined e.g. EIGEN_DONT_ALIGN) so it needs to be done before we do anything with vectorization.
88#include "src/Core/util/Macros.h"
89
90// Disable the ipa-cp-clone optimization flag with MinGW 6.x or newer (enabled by default with -O3)
91// See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=556 for details.
92#if EIGEN_COMP_MINGW && EIGEN_GNUC_AT_LEAST(4,6)
93 #pragma GCC optimize ("-fno-ipa-cp-clone")
94#endif
95
96#include <complex>
97
98// this include file manages BLAS and MKL related macros
99// and inclusion of their respective header files
100#include "src/Core/util/MKL_support.h"
101
102// if alignment is disabled, then disable vectorization. Note: EIGEN_MAX_ALIGN_BYTES is the proper check, it takes into
103// account both the user's will (EIGEN_MAX_ALIGN_BYTES,EIGEN_DONT_ALIGN) and our own platform checks
104#if EIGEN_MAX_ALIGN_BYTES==0
105 #ifndef EIGEN_DONT_VECTORIZE
106 #define EIGEN_DONT_VECTORIZE
107 #endif
108#endif
109
110#if EIGEN_COMP_MSVC
111 #include <malloc.h> // for _aligned_malloc -- need it regardless of whether vectorization is enabled
112 #if (EIGEN_COMP_MSVC >= 1500) // 2008 or later
113 // Remember that usage of defined() in a #define is undefined by the standard.
114 // a user reported that in 64-bit mode, MSVC doesn't care to define _M_IX86_FP.
115 #if (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64
116 #define EIGEN_SSE2_ON_MSVC_2008_OR_LATER
117 #endif
118 #endif
119#else
120 // Remember that usage of defined() in a #define is undefined by the standard
121 #if (defined __SSE2__) && ( (!EIGEN_COMP_GNUC) || EIGEN_COMP_ICC || EIGEN_GNUC_AT_LEAST(4,2) )
122 #define EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC
123 #endif
124#endif
125
126#ifndef EIGEN_DONT_VECTORIZE
127
128 #if defined (EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC) || defined(EIGEN_SSE2_ON_MSVC_2008_OR_LATER)
129
130 // Defines symbols for compile-time detection of which instructions are
131 // used.
132 // EIGEN_VECTORIZE_YY is defined if and only if the instruction set YY is used
133 #define EIGEN_VECTORIZE
134 #define EIGEN_VECTORIZE_SSE
135 #define EIGEN_VECTORIZE_SSE2
136
137 // Detect sse3/ssse3/sse4:
138 // gcc and icc defines __SSE3__, ...
139 // there is no way to know about this on msvc. You can define EIGEN_VECTORIZE_SSE* if you
140 // want to force the use of those instructions with msvc.
141 #ifdef __SSE3__
142 #define EIGEN_VECTORIZE_SSE3
143 #endif
144 #ifdef __SSSE3__
145 #define EIGEN_VECTORIZE_SSSE3
146 #endif
147 #ifdef __SSE4_1__
148 #define EIGEN_VECTORIZE_SSE4_1
149 #endif
150 #ifdef __SSE4_2__
151 #define EIGEN_VECTORIZE_SSE4_2
152 #endif
153 #ifdef __AVX__
154 #define EIGEN_VECTORIZE_AVX
155 #define EIGEN_VECTORIZE_SSE3
156 #define EIGEN_VECTORIZE_SSSE3
157 #define EIGEN_VECTORIZE_SSE4_1
158 #define EIGEN_VECTORIZE_SSE4_2
159 #endif
160 #ifdef __AVX2__
161 #define EIGEN_VECTORIZE_AVX2
162 #endif
163 #ifdef __FMA__
164 #define EIGEN_VECTORIZE_FMA
165 #endif
166 #if defined(__AVX512F__) && defined(EIGEN_ENABLE_AVX512)
167 #define EIGEN_VECTORIZE_AVX512
168 #define EIGEN_VECTORIZE_AVX2
169 #define EIGEN_VECTORIZE_AVX
170 #define EIGEN_VECTORIZE_FMA
171 #ifdef __AVX512DQ__
172 #define EIGEN_VECTORIZE_AVX512DQ
173 #endif
174 #ifdef __AVX512ER__
175 #define EIGEN_VECTORIZE_AVX512ER
176 #endif
177 #endif
178
179 // include files
180
181 // This extern "C" works around a MINGW-w64 compilation issue
182 // https://sourceforge.net/tracker/index.php?func=detail&aid=3018394&group_id=202880&atid=983354
183 // In essence, intrin.h is included by windows.h and also declares intrinsics (just as emmintrin.h etc. below do).
184 // However, intrin.h uses an extern "C" declaration, and g++ thus complains of duplicate declarations
185 // with conflicting linkage. The linkage for intrinsics doesn't matter, but at that stage the compiler doesn't know;
186 // so, to avoid compile errors when windows.h is included after Eigen/Core, ensure intrinsics are extern "C" here too.
187 // notice that since these are C headers, the extern "C" is theoretically needed anyways.
188 extern "C" {
189 // In theory we should only include immintrin.h and not the other *mmintrin.h header files directly.
190 // Doing so triggers some issues with ICC. However old gcc versions seems to not have this file, thus:
191 #if EIGEN_COMP_ICC >= 1110
192 #include <immintrin.h>
193 #else
194 #include <mmintrin.h>
195 #include <emmintrin.h>
196 #include <xmmintrin.h>
197 #ifdef EIGEN_VECTORIZE_SSE3
198 #include <pmmintrin.h>
199 #endif
200 #ifdef EIGEN_VECTORIZE_SSSE3
201 #include <tmmintrin.h>
202 #endif
203 #ifdef EIGEN_VECTORIZE_SSE4_1
204 #include <smmintrin.h>
205 #endif
206 #ifdef EIGEN_VECTORIZE_SSE4_2
207 #include <nmmintrin.h>
208 #endif
209 #if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512)
210 #include <immintrin.h>
211 #endif
212 #endif
213 } // end extern "C"
214 #elif defined __VSX__
215 #define EIGEN_VECTORIZE
216 #define EIGEN_VECTORIZE_VSX
217 #include <altivec.h>
218 // We need to #undef all these ugly tokens defined in <altivec.h>
219 // => use __vector instead of vector
220 #undef bool
221 #undef vector
222 #undef pixel
223 #elif defined __ALTIVEC__
224 #define EIGEN_VECTORIZE
225 #define EIGEN_VECTORIZE_ALTIVEC
226 #include <altivec.h>
227 // We need to #undef all these ugly tokens defined in <altivec.h>
228 // => use __vector instead of vector
229 #undef bool
230 #undef vector
231 #undef pixel
232 #elif (defined __ARM_NEON) || (defined __ARM_NEON__)
233 #define EIGEN_VECTORIZE
234 #define EIGEN_VECTORIZE_NEON
235 #include <arm_neon.h>
236 #elif (defined __s390x__ && defined __VEC__)
237 #define EIGEN_VECTORIZE
238 #define EIGEN_VECTORIZE_ZVECTOR
239 #include <vecintrin.h>
240 #endif
241#endif
242
243#if defined(__F16C__) && !defined(EIGEN_COMP_CLANG)
244 // We can use the optimized fp16 to float and float to fp16 conversion routines
245 #define EIGEN_HAS_FP16_C
246#endif
247
248#if defined __CUDACC__
249 #define EIGEN_VECTORIZE_CUDA
250 #include <vector_types.h>
251 #if EIGEN_CUDACC_VER >= 70500
252 #define EIGEN_HAS_CUDA_FP16
253 #endif
254#endif
255
256#if defined EIGEN_HAS_CUDA_FP16
257 #include <host_defines.h>
258 #include <cuda_fp16.h>
259#endif
260
261#if (defined _OPENMP) && (!defined EIGEN_DONT_PARALLELIZE)
262 #define EIGEN_HAS_OPENMP
263#endif
264
265#ifdef EIGEN_HAS_OPENMP
266#include <omp.h>
267#endif
268
269// MSVC for windows mobile does not have the errno.h file
270#if !(EIGEN_COMP_MSVC && EIGEN_OS_WINCE) && !EIGEN_COMP_ARM
271#define EIGEN_HAS_ERRNO
272#endif
273
274#ifdef EIGEN_HAS_ERRNO
275#include <cerrno>
276#endif
277#include <cstddef>
278#include <cstdlib>
279#include <cmath>
280#include <cassert>
281#include <functional>
282#include <iosfwd>
283#include <cstring>
284#include <string>
285#include <limits>
286#include <climits> // for CHAR_BIT
287// for min/max:
288#include <algorithm>
289
290// for std::is_nothrow_move_assignable
291#ifdef EIGEN_INCLUDE_TYPE_TRAITS
292#include <type_traits>
293#endif
294
295// for outputting debug info
296#ifdef EIGEN_DEBUG_ASSIGN
297#include <iostream>
298#endif
299
300// required for __cpuid, needs to be included after cmath
301#if EIGEN_COMP_MSVC && EIGEN_ARCH_i386_OR_x86_64 && !EIGEN_OS_WINCE
302 #include <intrin.h>
303#endif
304
305/** \brief Namespace containing all symbols from the %Eigen library. */
306namespace Eigen {
307
308inline static const char *SimdInstructionSetsInUse(void) {
309#if defined(EIGEN_VECTORIZE_AVX512)
310 return "AVX512, FMA, AVX2, AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
311#elif defined(EIGEN_VECTORIZE_AVX)
312 return "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
313#elif defined(EIGEN_VECTORIZE_SSE4_2)
314 return "SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2";
315#elif defined(EIGEN_VECTORIZE_SSE4_1)
316 return "SSE, SSE2, SSE3, SSSE3, SSE4.1";
317#elif defined(EIGEN_VECTORIZE_SSSE3)
318 return "SSE, SSE2, SSE3, SSSE3";
319#elif defined(EIGEN_VECTORIZE_SSE3)
320 return "SSE, SSE2, SSE3";
321#elif defined(EIGEN_VECTORIZE_SSE2)
322 return "SSE, SSE2";
323#elif defined(EIGEN_VECTORIZE_ALTIVEC)
324 return "AltiVec";
325#elif defined(EIGEN_VECTORIZE_VSX)
326 return "VSX";
327#elif defined(EIGEN_VECTORIZE_NEON)
328 return "ARM NEON";
329#elif defined(EIGEN_VECTORIZE_ZVECTOR)
330 return "S390X ZVECTOR";
331#else
332 return "None";
333#endif
334}
335
336} // end namespace Eigen
337
338#if defined EIGEN2_SUPPORT_STAGE40_FULL_EIGEN3_STRICTNESS || defined EIGEN2_SUPPORT_STAGE30_FULL_EIGEN3_API || defined EIGEN2_SUPPORT_STAGE20_RESOLVE_API_CONFLICTS || defined EIGEN2_SUPPORT_STAGE10_FULL_EIGEN2_API || defined EIGEN2_SUPPORT
339// This will generate an error message:
340#error Eigen2-support is only available up to version 3.2. Please go to "http://eigen.tuxfamily.org/index.php?title=Eigen2" for further information
341#endif
342
343namespace Eigen {
344
345// we use size_t frequently and we'll never remember to prepend it with std:: everytime just to
346// ensure QNX/QCC support
347using std::size_t;
348// gcc 4.6.0 wants std:: for ptrdiff_t
349using std::ptrdiff_t;
350
351}
352
353/** \defgroup Core_Module Core module
354 * This is the main module of Eigen providing dense matrix and vector support
355 * (both fixed and dynamic size) with all the features corresponding to a BLAS library
356 * and much more...
357 *
358 * \code
359 * #include <Eigen/Core>
360 * \endcode
361 */
362
363#include "src/Core/util/Constants.h"
364#include "src/Core/util/Meta.h"
365#include "src/Core/util/ForwardDeclarations.h"
366#include "src/Core/util/StaticAssert.h"
367#include "src/Core/util/XprHelper.h"
368#include "src/Core/util/Memory.h"
369
370#include "src/Core/NumTraits.h"
371#include "src/Core/MathFunctions.h"
372#include "src/Core/GenericPacketMath.h"
373#include "src/Core/MathFunctionsImpl.h"
374#include "src/Core/arch/Default/ConjHelper.h"
375
376#if defined EIGEN_VECTORIZE_AVX512
377 #include "src/Core/arch/SSE/PacketMath.h"
378 #include "src/Core/arch/AVX/PacketMath.h"
379 #include "src/Core/arch/AVX512/PacketMath.h"
380 #include "src/Core/arch/AVX512/MathFunctions.h"
381#elif defined EIGEN_VECTORIZE_AVX
382 // Use AVX for floats and doubles, SSE for integers
383 #include "src/Core/arch/SSE/PacketMath.h"
384 #include "src/Core/arch/SSE/Complex.h"
385 #include "src/Core/arch/SSE/MathFunctions.h"
386 #include "src/Core/arch/AVX/PacketMath.h"
387 #include "src/Core/arch/AVX/MathFunctions.h"
388 #include "src/Core/arch/AVX/Complex.h"
389 #include "src/Core/arch/AVX/TypeCasting.h"
390 #include "src/Core/arch/SSE/TypeCasting.h"
391#elif defined EIGEN_VECTORIZE_SSE
392 #include "src/Core/arch/SSE/PacketMath.h"
393 #include "src/Core/arch/SSE/MathFunctions.h"
394 #include "src/Core/arch/SSE/Complex.h"
395 #include "src/Core/arch/SSE/TypeCasting.h"
396#elif defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX)
397 #include "src/Core/arch/AltiVec/PacketMath.h"
398 #include "src/Core/arch/AltiVec/MathFunctions.h"
399 #include "src/Core/arch/AltiVec/Complex.h"
400#elif defined EIGEN_VECTORIZE_NEON
401 #include "src/Core/arch/NEON/PacketMath.h"
402 #include "src/Core/arch/NEON/MathFunctions.h"
403 #include "src/Core/arch/NEON/Complex.h"
404#elif defined EIGEN_VECTORIZE_ZVECTOR
405 #include "src/Core/arch/ZVector/PacketMath.h"
406 #include "src/Core/arch/ZVector/MathFunctions.h"
407 #include "src/Core/arch/ZVector/Complex.h"
408#endif
409
410// Half float support
411#include "src/Core/arch/CUDA/Half.h"
412#include "src/Core/arch/CUDA/PacketMathHalf.h"
413#include "src/Core/arch/CUDA/TypeCasting.h"
414
415#if defined EIGEN_VECTORIZE_CUDA
416 #include "src/Core/arch/CUDA/PacketMath.h"
417 #include "src/Core/arch/CUDA/MathFunctions.h"
418#endif
419
420#include "src/Core/arch/Default/Settings.h"
421
422#include "src/Core/functors/TernaryFunctors.h"
423#include "src/Core/functors/BinaryFunctors.h"
424#include "src/Core/functors/UnaryFunctors.h"
425#include "src/Core/functors/NullaryFunctors.h"
426#include "src/Core/functors/StlFunctors.h"
427#include "src/Core/functors/AssignmentFunctors.h"
428
429// Specialized functors to enable the processing of complex numbers
430// on CUDA devices
431#include "src/Core/arch/CUDA/Complex.h"
432
433#include "src/Core/IO.h"
434#include "src/Core/DenseCoeffsBase.h"
435#include "src/Core/DenseBase.h"
436#include "src/Core/MatrixBase.h"
437#include "src/Core/EigenBase.h"
438
439#include "src/Core/Product.h"
440#include "src/Core/CoreEvaluators.h"
441#include "src/Core/AssignEvaluator.h"
442
443#ifndef EIGEN_PARSED_BY_DOXYGEN // work around Doxygen bug triggered by Assign.h r814874
444 // at least confirmed with Doxygen 1.5.5 and 1.5.6
445 #include "src/Core/Assign.h"
446#endif
447
448#include "src/Core/ArrayBase.h"
449#include "src/Core/util/BlasUtil.h"
450#include "src/Core/DenseStorage.h"
451#include "src/Core/NestByValue.h"
452
453// #include "src/Core/ForceAlignedAccess.h"
454
455#include "src/Core/ReturnByValue.h"
456#include "src/Core/NoAlias.h"
457#include "src/Core/PlainObjectBase.h"
458#include "src/Core/Matrix.h"
459#include "src/Core/Array.h"
460#include "src/Core/CwiseTernaryOp.h"
461#include "src/Core/CwiseBinaryOp.h"
462#include "src/Core/CwiseUnaryOp.h"
463#include "src/Core/CwiseNullaryOp.h"
464#include "src/Core/CwiseUnaryView.h"
465#include "src/Core/SelfCwiseBinaryOp.h"
466#include "src/Core/Dot.h"
467#include "src/Core/StableNorm.h"
468#include "src/Core/Stride.h"
469#include "src/Core/MapBase.h"
470#include "src/Core/Map.h"
471#include "src/Core/Ref.h"
472#include "src/Core/Block.h"
473#include "src/Core/VectorBlock.h"
474#include "src/Core/Transpose.h"
475#include "src/Core/DiagonalMatrix.h"
476#include "src/Core/Diagonal.h"
477#include "src/Core/DiagonalProduct.h"
478#include "src/Core/Redux.h"
479#include "src/Core/Visitor.h"
480#include "src/Core/Fuzzy.h"
481#include "src/Core/Swap.h"
482#include "src/Core/CommaInitializer.h"
483#include "src/Core/GeneralProduct.h"
484#include "src/Core/Solve.h"
485#include "src/Core/Inverse.h"
486#include "src/Core/SolverBase.h"
487#include "src/Core/PermutationMatrix.h"
488#include "src/Core/Transpositions.h"
489#include "src/Core/TriangularMatrix.h"
490#include "src/Core/SelfAdjointView.h"
491#include "src/Core/products/GeneralBlockPanelKernel.h"
492#include "src/Core/products/Parallelizer.h"
493#include "src/Core/ProductEvaluators.h"
494#include "src/Core/products/GeneralMatrixVector.h"
495#include "src/Core/products/GeneralMatrixMatrix.h"
496#include "src/Core/SolveTriangular.h"
497#include "src/Core/products/GeneralMatrixMatrixTriangular.h"
498#include "src/Core/products/SelfadjointMatrixVector.h"
499#include "src/Core/products/SelfadjointMatrixMatrix.h"
500#include "src/Core/products/SelfadjointProduct.h"
501#include "src/Core/products/SelfadjointRank2Update.h"
502#include "src/Core/products/TriangularMatrixVector.h"
503#include "src/Core/products/TriangularMatrixMatrix.h"
504#include "src/Core/products/TriangularSolverMatrix.h"
505#include "src/Core/products/TriangularSolverVector.h"
506#include "src/Core/BandMatrix.h"
507#include "src/Core/CoreIterators.h"
508#include "src/Core/ConditionEstimator.h"
509
510#include "src/Core/BooleanRedux.h"
511#include "src/Core/Select.h"
512#include "src/Core/VectorwiseOp.h"
513#include "src/Core/Random.h"
514#include "src/Core/Replicate.h"
515#include "src/Core/Reverse.h"
516#include "src/Core/ArrayWrapper.h"
517
518#ifdef EIGEN_USE_BLAS
519#include "src/Core/products/GeneralMatrixMatrix_BLAS.h"
520#include "src/Core/products/GeneralMatrixVector_BLAS.h"
521#include "src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h"
522#include "src/Core/products/SelfadjointMatrixMatrix_BLAS.h"
523#include "src/Core/products/SelfadjointMatrixVector_BLAS.h"
524#include "src/Core/products/TriangularMatrixMatrix_BLAS.h"
525#include "src/Core/products/TriangularMatrixVector_BLAS.h"
526#include "src/Core/products/TriangularSolverMatrix_BLAS.h"
527#endif // EIGEN_USE_BLAS
528
529#ifdef EIGEN_USE_MKL_VML
530#include "src/Core/Assign_MKL.h"
531#endif
532
533#include "src/Core/GlobalFunctions.h"
534
535#include "src/Core/util/ReenableStupidWarnings.h"
536
537#endif // EIGEN_CORE_H
538