12#ifndef EIGEN_ASSIGN_EVALUATOR_H
13#define EIGEN_ASSIGN_EVALUATOR_H
27template <
typename DstEvaluator,
typename SrcEvaluator,
typename AssignFunc,
int MaxPacketSize = -1>
28struct copy_using_evaluator_traits
30 typedef typename DstEvaluator::XprType Dst;
31 typedef typename Dst::Scalar DstScalar;
34 DstFlags = DstEvaluator::Flags,
35 SrcFlags = SrcEvaluator::Flags
40 DstAlignment = DstEvaluator::Alignment,
41 SrcAlignment = SrcEvaluator::Alignment,
43 JointAlignment = EIGEN_PLAIN_ENUM_MIN(DstAlignment,SrcAlignment)
48 InnerSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::SizeAtCompileTime)
49 : int(DstFlags)&
RowMajorBit ? int(Dst::ColsAtCompileTime)
50 : int(Dst::RowsAtCompileTime),
51 InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime)
52 : int(DstFlags)&
RowMajorBit ? int(Dst::MaxColsAtCompileTime)
53 : int(Dst::MaxRowsAtCompileTime),
54 RestrictedInnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(InnerSize,MaxPacketSize),
55 RestrictedLinearSize = EIGEN_SIZE_MIN_PREFER_FIXED(Dst::SizeAtCompileTime,MaxPacketSize),
56 OuterStride = int(outer_stride_at_compile_time<Dst>::ret),
57 MaxSizeAtCompileTime = Dst::SizeAtCompileTime
61 typedef typename find_best_packet<DstScalar,RestrictedLinearSize>::type LinearPacketType;
62 typedef typename find_best_packet<DstScalar,RestrictedInnerSize>::type InnerPacketType;
65 LinearPacketSize = unpacket_traits<LinearPacketType>::size,
66 InnerPacketSize = unpacket_traits<InnerPacketType>::size
71 LinearRequiredAlignment = unpacket_traits<LinearPacketType>::alignment,
72 InnerRequiredAlignment = unpacket_traits<InnerPacketType>::alignment
79 StorageOrdersAgree = (int(DstIsRowMajor) == int(SrcIsRowMajor)),
80 MightVectorize =
bool(StorageOrdersAgree)
82 &&
bool(functor_traits<AssignFunc>::PacketAccess),
83 MayInnerVectorize = MightVectorize
84 && int(InnerSize)!=
Dynamic && int(InnerSize)%int(InnerPacketSize)==0
85 && int(OuterStride)!=
Dynamic && int(OuterStride)%int(InnerPacketSize)==0
86 && (EIGEN_UNALIGNED_VECTORIZE || int(JointAlignment)>=int(InnerRequiredAlignment)),
87 MayLinearize =
bool(StorageOrdersAgree) && (int(DstFlags) & int(SrcFlags) &
LinearAccessBit),
88 MayLinearVectorize =
bool(MightVectorize) && bool(MayLinearize) && bool(DstHasDirectAccess)
89 && (EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment)>=int(LinearRequiredAlignment)) || MaxSizeAtCompileTime ==
Dynamic),
92 MaySliceVectorize = bool(MightVectorize) && bool(DstHasDirectAccess)
93 && (int(InnerMaxSize)==
Dynamic || int(InnerMaxSize)>=(EIGEN_UNALIGNED_VECTORIZE?InnerPacketSize:(3*InnerPacketSize)))
102 Traversal = int(Dst::SizeAtCompileTime) == 0 ? int(AllAtOnceTraversal)
103 : (int(MayLinearVectorize) && (LinearPacketSize>InnerPacketSize)) ? int(LinearVectorizedTraversal)
104 : int(MayInnerVectorize) ? int(InnerVectorizedTraversal)
105 : int(MayLinearVectorize) ? int(LinearVectorizedTraversal)
106 : int(MaySliceVectorize) ? int(SliceVectorizedTraversal)
107 : int(MayLinearize) ? int(LinearTraversal)
108 : int(DefaultTraversal),
109 Vectorized = int(Traversal) == InnerVectorizedTraversal
110 || int(Traversal) == LinearVectorizedTraversal
111 || int(Traversal) == SliceVectorizedTraversal
114 typedef typename conditional<int(Traversal)==LinearVectorizedTraversal, LinearPacketType, InnerPacketType>::type PacketType;
118 ActualPacketSize = int(Traversal)==LinearVectorizedTraversal ? LinearPacketSize
119 : Vectorized ? InnerPacketSize
121 UnrollingLimit = EIGEN_UNROLLING_LIMIT * ActualPacketSize,
122 MayUnrollCompletely = int(Dst::SizeAtCompileTime) !=
Dynamic
123 && int(Dst::SizeAtCompileTime) * (int(DstEvaluator::CoeffReadCost)+int(SrcEvaluator::CoeffReadCost)) <=
int(UnrollingLimit),
124 MayUnrollInner = int(InnerSize) !=
Dynamic
125 && int(InnerSize) * (int(DstEvaluator::CoeffReadCost)+int(SrcEvaluator::CoeffReadCost)) <=
int(UnrollingLimit)
130 Unrolling = (int(Traversal) == int(InnerVectorizedTraversal) || int(Traversal) == int(DefaultTraversal))
132 int(MayUnrollCompletely) ? int(CompleteUnrolling)
133 : int(MayUnrollInner) ? int(InnerUnrolling)
136 : int(Traversal) == int(LinearVectorizedTraversal)
137 ? ( bool(MayUnrollCompletely) && ( EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment)>=int(LinearRequiredAlignment)))
138 ? int(CompleteUnrolling)
140 : int(Traversal) == int(LinearTraversal)
141 ? ( bool(MayUnrollCompletely) ? int(CompleteUnrolling)
143#if EIGEN_UNALIGNED_VECTORIZE
144 : int(Traversal) == int(SliceVectorizedTraversal)
145 ? ( bool(MayUnrollInner) ? int(InnerUnrolling)
151#ifdef EIGEN_DEBUG_ASSIGN
154 std::cerr <<
"DstXpr: " <<
typeid(
typename DstEvaluator::XprType).name() << std::endl;
155 std::cerr <<
"SrcXpr: " <<
typeid(
typename SrcEvaluator::XprType).name() << std::endl;
156 std::cerr.setf(std::ios::hex, std::ios::basefield);
157 std::cerr <<
"DstFlags" <<
" = " << DstFlags <<
" (" << demangle_flags(DstFlags) <<
" )" << std::endl;
158 std::cerr <<
"SrcFlags" <<
" = " << SrcFlags <<
" (" << demangle_flags(SrcFlags) <<
" )" << std::endl;
159 std::cerr.unsetf(std::ios::hex);
160 EIGEN_DEBUG_VAR(DstAlignment)
161 EIGEN_DEBUG_VAR(SrcAlignment)
162 EIGEN_DEBUG_VAR(LinearRequiredAlignment)
163 EIGEN_DEBUG_VAR(InnerRequiredAlignment)
164 EIGEN_DEBUG_VAR(JointAlignment)
165 EIGEN_DEBUG_VAR(InnerSize)
166 EIGEN_DEBUG_VAR(InnerMaxSize)
167 EIGEN_DEBUG_VAR(LinearPacketSize)
168 EIGEN_DEBUG_VAR(InnerPacketSize)
169 EIGEN_DEBUG_VAR(ActualPacketSize)
170 EIGEN_DEBUG_VAR(StorageOrdersAgree)
171 EIGEN_DEBUG_VAR(MightVectorize)
172 EIGEN_DEBUG_VAR(MayLinearize)
173 EIGEN_DEBUG_VAR(MayInnerVectorize)
174 EIGEN_DEBUG_VAR(MayLinearVectorize)
175 EIGEN_DEBUG_VAR(MaySliceVectorize)
176 std::cerr <<
"Traversal" <<
" = " << Traversal <<
" (" << demangle_traversal(Traversal) <<
")" << std::endl;
177 EIGEN_DEBUG_VAR(SrcEvaluator::CoeffReadCost)
178 EIGEN_DEBUG_VAR(DstEvaluator::CoeffReadCost)
179 EIGEN_DEBUG_VAR(Dst::SizeAtCompileTime)
180 EIGEN_DEBUG_VAR(UnrollingLimit)
181 EIGEN_DEBUG_VAR(MayUnrollCompletely)
182 EIGEN_DEBUG_VAR(MayUnrollInner)
183 std::cerr <<
"Unrolling" <<
" = " << Unrolling <<
" (" << demangle_unrolling(Unrolling) <<
")" << std::endl;
184 std::cerr << std::endl;
197template<
typename Kernel,
int Index,
int Stop>
198struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling
201 typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
202 typedef typename DstEvaluatorType::XprType DstXprType;
205 outer =
Index / DstXprType::InnerSizeAtCompileTime,
206 inner =
Index % DstXprType::InnerSizeAtCompileTime
209 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
211 kernel.assignCoeffByOuterInner(outer, inner);
212 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Index+1, Stop>::run(kernel);
216template<
typename Kernel,
int Stop>
217struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Stop, Stop>
219 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&) { }
222template<
typename Kernel,
int Index_,
int Stop>
223struct copy_using_evaluator_DefaultTraversal_InnerUnrolling
225 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel, Index outer)
227 kernel.assignCoeffByOuterInner(outer, Index_);
228 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Index_+1, Stop>::run(kernel, outer);
232template<
typename Kernel,
int Stop>
233struct copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Stop, Stop>
235 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&, Index) { }
242template<
typename Kernel,
int Index,
int Stop>
243struct copy_using_evaluator_LinearTraversal_CompleteUnrolling
245 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel& kernel)
247 kernel.assignCoeff(Index);
248 copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Index+1, Stop>::run(kernel);
252template<
typename Kernel,
int Stop>
253struct copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Stop, Stop>
255 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&) { }
262template<
typename Kernel,
int Index,
int Stop>
263struct copy_using_evaluator_innervec_CompleteUnrolling
266 typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
267 typedef typename DstEvaluatorType::XprType DstXprType;
268 typedef typename Kernel::PacketType PacketType;
271 outer =
Index / DstXprType::InnerSizeAtCompileTime,
272 inner =
Index % DstXprType::InnerSizeAtCompileTime,
273 SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
274 DstAlignment = Kernel::AssignmentTraits::DstAlignment
277 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
279 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
280 enum { NextIndex =
Index + unpacket_traits<PacketType>::size };
281 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, NextIndex, Stop>::run(kernel);
285template<
typename Kernel,
int Stop>
286struct copy_using_evaluator_innervec_CompleteUnrolling<Kernel, Stop, Stop>
288 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&) { }
291template<
typename Kernel,
int Index_,
int Stop,
int SrcAlignment,
int DstAlignment>
292struct copy_using_evaluator_innervec_InnerUnrolling
294 typedef typename Kernel::PacketType PacketType;
295 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel, Index outer)
297 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, Index_);
298 enum { NextIndex = Index_ + unpacket_traits<PacketType>::size };
299 copy_using_evaluator_innervec_InnerUnrolling<Kernel, NextIndex, Stop, SrcAlignment, DstAlignment>::run(kernel, outer);
303template<
typename Kernel,
int Stop,
int SrcAlignment,
int DstAlignment>
304struct copy_using_evaluator_innervec_InnerUnrolling<Kernel, Stop, Stop, SrcAlignment, DstAlignment>
306 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &, Index) { }
315template<
typename Kernel,
316 int Traversal = Kernel::AssignmentTraits::Traversal,
317 int Unrolling = Kernel::AssignmentTraits::Unrolling>
318struct dense_assignment_loop;
325template<
typename Kernel,
int Unrolling>
326struct dense_assignment_loop<Kernel, AllAtOnceTraversal, Unrolling>
328 EIGEN_DEVICE_FUNC
static void EIGEN_STRONG_INLINE run(Kernel& )
330 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
331 EIGEN_STATIC_ASSERT(
int(DstXprType::SizeAtCompileTime) == 0,
332 EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT)
340template<
typename Kernel>
341struct dense_assignment_loop<Kernel, DefaultTraversal, NoUnrolling>
343 EIGEN_DEVICE_FUNC
static void EIGEN_STRONG_INLINE run(Kernel &kernel)
345 for(Index outer = 0; outer < kernel.outerSize(); ++outer) {
346 for(Index inner = 0; inner < kernel.innerSize(); ++inner) {
347 kernel.assignCoeffByOuterInner(outer, inner);
353template<
typename Kernel>
354struct dense_assignment_loop<Kernel, DefaultTraversal, CompleteUnrolling>
356 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
358 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
359 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
363template<
typename Kernel>
364struct dense_assignment_loop<Kernel, DefaultTraversal, InnerUnrolling>
366 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
368 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
370 const Index outerSize = kernel.outerSize();
371 for(Index outer = 0; outer < outerSize; ++outer)
372 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime>::run(kernel, outer);
384template <
bool IsAligned = false>
385struct unaligned_dense_assignment_loop
388 template <
typename Kernel>
389 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&, Index, Index) {}
393struct unaligned_dense_assignment_loop<false>
399 template <
typename Kernel>
400 static EIGEN_DONT_INLINE
void run(Kernel &kernel,
404 template <
typename Kernel>
405 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel,
410 for (Index index = start; index < end; ++index)
411 kernel.assignCoeff(index);
415template<
typename Kernel>
416struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, NoUnrolling>
418 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
420 const Index size = kernel.size();
421 typedef typename Kernel::Scalar Scalar;
422 typedef typename Kernel::PacketType PacketType;
424 requestedAlignment = Kernel::AssignmentTraits::LinearRequiredAlignment,
425 packetSize = unpacket_traits<PacketType>::size,
426 dstIsAligned = int(Kernel::AssignmentTraits::DstAlignment)>=int(requestedAlignment),
427 dstAlignment = packet_traits<Scalar>::AlignedOnScalar ? int(requestedAlignment)
428 : int(Kernel::AssignmentTraits::DstAlignment),
429 srcAlignment = Kernel::AssignmentTraits::JointAlignment
431 const Index alignedStart = dstIsAligned ? 0 : internal::first_aligned<requestedAlignment>(kernel.dstDataPtr(), size);
432 const Index alignedEnd = alignedStart + ((size-alignedStart)/packetSize)*packetSize;
434 unaligned_dense_assignment_loop<dstIsAligned!=0>::run(kernel, 0, alignedStart);
436 for(Index index = alignedStart; index < alignedEnd; index += packetSize)
437 kernel.template assignPacket<dstAlignment, srcAlignment, PacketType>(index);
439 unaligned_dense_assignment_loop<>::run(kernel, alignedEnd, size);
443template<
typename Kernel>
444struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, CompleteUnrolling>
446 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
448 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
449 typedef typename Kernel::PacketType PacketType;
451 enum { size = DstXprType::SizeAtCompileTime,
452 packetSize =unpacket_traits<PacketType>::size,
453 alignedSize = (int(size)/packetSize)*packetSize };
455 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, alignedSize>::run(kernel);
456 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, alignedSize, size>::run(kernel);
464template<
typename Kernel>
465struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, NoUnrolling>
467 typedef typename Kernel::PacketType PacketType;
469 SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
470 DstAlignment = Kernel::AssignmentTraits::DstAlignment
472 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
474 const Index innerSize = kernel.innerSize();
475 const Index outerSize = kernel.outerSize();
476 const Index packetSize = unpacket_traits<PacketType>::size;
477 for(Index outer = 0; outer < outerSize; ++outer)
478 for(Index inner = 0; inner < innerSize; inner+=packetSize)
479 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
483template<
typename Kernel>
484struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, CompleteUnrolling>
486 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
488 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
489 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
493template<
typename Kernel>
494struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, InnerUnrolling>
496 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
498 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
499 typedef typename Kernel::AssignmentTraits Traits;
500 const Index outerSize = kernel.outerSize();
501 for(Index outer = 0; outer < outerSize; ++outer)
502 copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime,
503 Traits::SrcAlignment, Traits::DstAlignment>::run(kernel, outer);
511template<
typename Kernel>
512struct dense_assignment_loop<Kernel, LinearTraversal, NoUnrolling>
514 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
516 const Index size = kernel.size();
517 for(Index i = 0; i < size; ++i)
518 kernel.assignCoeff(i);
522template<
typename Kernel>
523struct dense_assignment_loop<Kernel, LinearTraversal, CompleteUnrolling>
525 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
527 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
528 copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
536template<
typename Kernel>
537struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, NoUnrolling>
539 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
541 typedef typename Kernel::Scalar Scalar;
542 typedef typename Kernel::PacketType PacketType;
544 packetSize = unpacket_traits<PacketType>::size,
545 requestedAlignment = int(Kernel::AssignmentTraits::InnerRequiredAlignment),
546 alignable = packet_traits<Scalar>::AlignedOnScalar || int(Kernel::AssignmentTraits::DstAlignment)>=
sizeof(Scalar),
547 dstIsAligned =
int(Kernel::AssignmentTraits::DstAlignment)>=int(requestedAlignment),
548 dstAlignment = alignable ? int(requestedAlignment)
549 : int(Kernel::AssignmentTraits::DstAlignment)
551 const Scalar *dst_ptr = kernel.dstDataPtr();
552 if((!
bool(dstIsAligned)) && (UIntPtr(dst_ptr) %
sizeof(Scalar))>0)
555 return dense_assignment_loop<Kernel,DefaultTraversal,NoUnrolling>::run(kernel);
557 const Index packetAlignedMask = packetSize - 1;
558 const Index innerSize = kernel.innerSize();
559 const Index outerSize = kernel.outerSize();
560 const Index alignedStep = alignable ? (packetSize - kernel.outerStride() % packetSize) & packetAlignedMask : 0;
561 Index alignedStart = ((!alignable) ||
bool(dstIsAligned)) ? 0 : internal::first_aligned<requestedAlignment>(dst_ptr, innerSize);
563 for(Index outer = 0; outer < outerSize; ++outer)
565 const Index alignedEnd = alignedStart + ((innerSize-alignedStart) & ~packetAlignedMask);
567 for(Index inner = 0; inner<alignedStart ; ++inner)
568 kernel.assignCoeffByOuterInner(outer, inner);
571 for(Index inner = alignedStart; inner<alignedEnd; inner+=packetSize)
572 kernel.template assignPacketByOuterInner<dstAlignment, Unaligned, PacketType>(outer, inner);
575 for(Index inner = alignedEnd; inner<innerSize ; ++inner)
576 kernel.assignCoeffByOuterInner(outer, inner);
578 alignedStart = numext::mini((alignedStart+alignedStep)%packetSize, innerSize);
583#if EIGEN_UNALIGNED_VECTORIZE
584template<
typename Kernel>
585struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, InnerUnrolling>
587 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
589 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
590 typedef typename Kernel::PacketType PacketType;
592 enum { innerSize = DstXprType::InnerSizeAtCompileTime,
593 packetSize =unpacket_traits<PacketType>::size,
594 vectorizableSize = (int(innerSize) / int(packetSize)) *
int(packetSize),
595 size = DstXprType::SizeAtCompileTime };
597 for(Index outer = 0; outer < kernel.outerSize(); ++outer)
599 copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, vectorizableSize, 0, 0>::run(kernel, outer);
600 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, vectorizableSize, innerSize>::run(kernel, outer);
617template<
typename DstEvaluatorTypeT,
typename SrcEvaluatorTypeT,
typename Functor,
int Version = Specialized>
618class generic_dense_assignment_kernel
621 typedef typename DstEvaluatorTypeT::XprType DstXprType;
622 typedef typename SrcEvaluatorTypeT::XprType SrcXprType;
625 typedef DstEvaluatorTypeT DstEvaluatorType;
626 typedef SrcEvaluatorTypeT SrcEvaluatorType;
627 typedef typename DstEvaluatorType::Scalar Scalar;
628 typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor> AssignmentTraits;
629 typedef typename AssignmentTraits::PacketType PacketType;
632 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
633 generic_dense_assignment_kernel(DstEvaluatorType &dst,
const SrcEvaluatorType &src,
const Functor &func, DstXprType& dstExpr)
634 : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr)
636 #ifdef EIGEN_DEBUG_ASSIGN
637 AssignmentTraits::debug();
641 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index size() const EIGEN_NOEXCEPT {
return m_dstExpr.size(); }
642 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index innerSize() const EIGEN_NOEXCEPT {
return m_dstExpr.innerSize(); }
643 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index outerSize() const EIGEN_NOEXCEPT {
return m_dstExpr.outerSize(); }
644 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index rows() const EIGEN_NOEXCEPT {
return m_dstExpr.rows(); }
645 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index cols() const EIGEN_NOEXCEPT {
return m_dstExpr.cols(); }
646 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index outerStride() const EIGEN_NOEXCEPT {
return m_dstExpr.outerStride(); }
648 EIGEN_DEVICE_FUNC DstEvaluatorType& dstEvaluator() EIGEN_NOEXCEPT {
return m_dst; }
649 EIGEN_DEVICE_FUNC
const SrcEvaluatorType& srcEvaluator() const EIGEN_NOEXCEPT {
return m_src; }
652 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignCoeff(Index row, Index col)
654 m_functor.assignCoeff(m_dst.coeffRef(row,col), m_src.coeff(row,col));
658 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignCoeff(Index index)
660 m_functor.assignCoeff(m_dst.coeffRef(index), m_src.coeff(index));
664 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignCoeffByOuterInner(Index outer, Index inner)
666 Index row = rowIndexByOuterInner(outer, inner);
667 Index col = colIndexByOuterInner(outer, inner);
668 assignCoeff(row, col);
672 template<
int StoreMode,
int LoadMode,
typename PacketType>
673 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacket(Index row, Index col)
675 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(row,col), m_src.template packet<LoadMode,PacketType>(row,col));
678 template<
int StoreMode,
int LoadMode,
typename PacketType>
679 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacket(Index index)
681 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(index), m_src.template packet<LoadMode,PacketType>(index));
684 template<
int StoreMode,
int LoadMode,
typename PacketType>
685 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacketByOuterInner(Index outer, Index inner)
687 Index row = rowIndexByOuterInner(outer, inner);
688 Index col = colIndexByOuterInner(outer, inner);
689 assignPacket<StoreMode,LoadMode,PacketType>(row, col);
692 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
Index rowIndexByOuterInner(Index outer, Index inner)
694 typedef typename DstEvaluatorType::ExpressionTraits Traits;
695 return int(Traits::RowsAtCompileTime) == 1 ? 0
696 : int(Traits::ColsAtCompileTime) == 1 ? inner
701 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
Index colIndexByOuterInner(Index outer, Index inner)
703 typedef typename DstEvaluatorType::ExpressionTraits Traits;
704 return int(Traits::ColsAtCompileTime) == 1 ? 0
705 : int(Traits::RowsAtCompileTime) == 1 ? inner
710 EIGEN_DEVICE_FUNC
const Scalar* dstDataPtr()
const
712 return m_dstExpr.data();
716 DstEvaluatorType& m_dst;
717 const SrcEvaluatorType& m_src;
718 const Functor &m_functor;
720 DstXprType& m_dstExpr;
727template<
typename DstEvaluatorTypeT,
typename SrcEvaluatorTypeT,
typename Functor>
728class restricted_packet_dense_assignment_kernel :
public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn>
731 typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn> Base;
733 typedef typename Base::Scalar Scalar;
734 typedef typename Base::DstXprType DstXprType;
735 typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, 4> AssignmentTraits;
736 typedef typename AssignmentTraits::PacketType PacketType;
738 EIGEN_DEVICE_FUNC restricted_packet_dense_assignment_kernel(DstEvaluatorTypeT &dst,
const SrcEvaluatorTypeT &src,
const Functor &func, DstXprType& dstExpr)
739 : Base(dst, src, func, dstExpr)
748template<
typename DstXprType,
typename SrcXprType,
typename Functor>
749EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
750void resize_if_allowed(DstXprType &dst,
const SrcXprType& src,
const Functor &)
752 EIGEN_ONLY_USED_FOR_DEBUG(dst);
753 EIGEN_ONLY_USED_FOR_DEBUG(src);
754 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
757template<
typename DstXprType,
typename SrcXprType,
typename T1,
typename T2>
758EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
759void resize_if_allowed(DstXprType &dst,
const SrcXprType& src,
const internal::assign_op<T1,T2> &)
761 Index dstRows = src.rows();
762 Index dstCols = src.cols();
763 if(((dst.rows()!=dstRows) || (dst.cols()!=dstCols)))
764 dst.resize(dstRows, dstCols);
765 eigen_assert(dst.rows() == dstRows && dst.cols() == dstCols);
768template<
typename DstXprType,
typename SrcXprType,
typename Functor>
769EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_dense_assignment_loop(DstXprType& dst,
const SrcXprType& src,
const Functor &func)
771 typedef evaluator<DstXprType> DstEvaluatorType;
772 typedef evaluator<SrcXprType> SrcEvaluatorType;
774 SrcEvaluatorType srcEvaluator(src);
778 resize_if_allowed(dst, src, func);
780 DstEvaluatorType dstEvaluator(dst);
782 typedef generic_dense_assignment_kernel<DstEvaluatorType,SrcEvaluatorType,Functor> Kernel;
783 Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
785 dense_assignment_loop<Kernel>::run(kernel);
789#ifndef EIGEN_GPU_COMPILE_PHASE
790template<
typename DstXprType>
791EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_dense_assignment_loop(DstXprType& dst,
const Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<typename DstXprType::Scalar>, DstXprType>& src,
const internal::assign_op<typename DstXprType::Scalar,typename DstXprType::Scalar>& func)
793 resize_if_allowed(dst, src, func);
794 std::fill_n(dst.data(), dst.size(), src.functor()());
798template<
typename DstXprType,
typename SrcXprType>
799EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_dense_assignment_loop(DstXprType& dst,
const SrcXprType& src)
801 call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>());
811template<
typename DstShape,
typename SrcShape>
struct AssignmentKind;
814struct Dense2Dense {};
815struct EigenBase2EigenBase {};
817template<
typename,
typename>
struct AssignmentKind {
typedef EigenBase2EigenBase Kind; };
818template<>
struct AssignmentKind<DenseShape,DenseShape> {
typedef Dense2Dense Kind; };
821template<
typename DstXprType,
typename SrcXprType,
typename Functor,
822 typename Kind =
typename AssignmentKind< typename evaluator_traits<DstXprType>::Shape ,
typename evaluator_traits<SrcXprType>::Shape >::Kind,
823 typename EnableIf =
void>
832template<
typename Dst,
typename Src>
833EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
834void call_assignment(Dst& dst,
const Src& src)
836 call_assignment(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
838template<
typename Dst,
typename Src>
839EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
840void call_assignment(
const Dst& dst,
const Src& src)
842 call_assignment(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
846template<
typename Dst,
typename Src,
typename Func>
847EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
848void call_assignment(Dst& dst,
const Src& src,
const Func& func,
typename enable_if< evaluator_assume_aliasing<Src>::value,
void*>::type = 0)
850 typename plain_matrix_type<Src>::type tmp(src);
851 call_assignment_no_alias(dst, tmp, func);
854template<
typename Dst,
typename Src,
typename Func>
855EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
856void call_assignment(Dst& dst,
const Src& src,
const Func& func,
typename enable_if<!evaluator_assume_aliasing<Src>::value,
void*>::type = 0)
858 call_assignment_no_alias(dst, src, func);
863template<
typename Dst,
template <
typename>
class StorageBase,
typename Src,
typename Func>
864EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
865void call_assignment(NoAlias<Dst,StorageBase>& dst,
const Src& src,
const Func& func)
867 call_assignment_no_alias(dst.expression(), src, func);
871template<
typename Dst,
typename Src,
typename Func>
872EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
873void call_assignment_no_alias(Dst& dst,
const Src& src,
const Func& func)
876 NeedToTranspose = ( (int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1)
877 || (
int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)
878 ) && int(Dst::SizeAtCompileTime) != 1
881 typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst>::type ActualDstTypeCleaned;
882 typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst&>::type ActualDstType;
883 ActualDstType actualDst(dst);
886 EIGEN_STATIC_ASSERT_LVALUE(Dst)
887 EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned,Src)
888 EIGEN_CHECK_BINARY_COMPATIBILIY(Func,
typename ActualDstTypeCleaned::Scalar,
typename Src::Scalar);
890 Assignment<ActualDstTypeCleaned,Src,Func>::run(actualDst, src, func);
893template<
typename Dst,
typename Src,
typename Func>
894EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
895void call_restricted_packet_assignment_no_alias(Dst& dst,
const Src& src,
const Func& func)
897 typedef evaluator<Dst> DstEvaluatorType;
898 typedef evaluator<Src> SrcEvaluatorType;
899 typedef restricted_packet_dense_assignment_kernel<DstEvaluatorType,SrcEvaluatorType,Func> Kernel;
901 EIGEN_STATIC_ASSERT_LVALUE(Dst)
902 EIGEN_CHECK_BINARY_COMPATIBILIY(Func,
typename Dst::Scalar,
typename Src::Scalar);
904 SrcEvaluatorType srcEvaluator(src);
905 resize_if_allowed(dst, src, func);
907 DstEvaluatorType dstEvaluator(dst);
908 Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
910 dense_assignment_loop<Kernel>::run(kernel);
913template<
typename Dst,
typename Src>
914EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
915void call_assignment_no_alias(Dst& dst,
const Src& src)
917 call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
920template<
typename Dst,
typename Src,
typename Func>
921EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
922void call_assignment_no_alias_no_transpose(Dst& dst,
const Src& src,
const Func& func)
925 EIGEN_STATIC_ASSERT_LVALUE(Dst)
926 EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src)
927 EIGEN_CHECK_BINARY_COMPATIBILIY(Func,
typename Dst::Scalar,
typename Src::Scalar);
929 Assignment<Dst,Src,Func>::run(dst, src, func);
931template<
typename Dst,
typename Src>
932EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
933void call_assignment_no_alias_no_transpose(Dst& dst,
const Src& src)
935 call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
939template<
typename Dst,
typename Src>
void check_for_aliasing(
const Dst &dst,
const Src &src);
944template<
typename DstXprType,
typename SrcXprType,
typename Functor,
typename Weak>
945struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Weak>
948 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const Functor &func)
950#ifndef EIGEN_NO_DEBUG
951 internal::check_for_aliasing(dst, src);
954 call_dense_assignment_loop(dst, src, func);
962template<
typename DstXprType,
typename SrcXprType,
typename Functor,
typename Weak>
963struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Weak>
966 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &)
968 Index dstRows = src.rows();
969 Index dstCols = src.cols();
970 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
971 dst.resize(dstRows, dstCols);
973 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
979 template<
typename SrcScalarType>
981 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const internal::add_assign_op<typename DstXprType::Scalar,SrcScalarType> &)
983 Index dstRows = src.rows();
984 Index dstCols = src.cols();
985 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
986 dst.resize(dstRows, dstCols);
988 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
992 template<
typename SrcScalarType>
994 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const internal::sub_assign_op<typename DstXprType::Scalar,SrcScalarType> &)
996 Index dstRows = src.rows();
997 Index dstCols = src.cols();
998 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
999 dst.resize(dstRows, dstCols);
1001 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
Generic expression of a matrix where all coefficients are defined by a functor.
Definition: CwiseNullaryOp.h:61
const unsigned int ActualPacketAccessBit
Definition: Constants.h:105
const unsigned int LinearAccessBit
Definition: Constants.h:130
const unsigned int DirectAccessBit
Definition: Constants.h:155
const unsigned int RowMajorBit
Definition: Constants.h:66
Namespace containing all symbols from the Eigen library.
Definition: Core:141
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:74
const int Dynamic
Definition: Constants.h:22