Please, help us to better know about our user community by answering the following short survey: https://forms.gle/wpyrxWi18ox9Z5ae9
Eigen  3.4.0
 
Loading...
Searching...
No Matches
NEON/TypeCasting.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2018 Rasmus Munk Larsen <rmlarsen@google.com>
5// Copyright (C) 2020 Antonio Sanchez <cantonios@google.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10
11#ifndef EIGEN_TYPE_CASTING_NEON_H
12#define EIGEN_TYPE_CASTING_NEON_H
13
14namespace Eigen {
15
16namespace internal {
17
18//==============================================================================
19// pcast, SrcType = float
20//==============================================================================
21template <>
22struct type_casting_traits<float, float> {
23 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
24};
25template <>
26EIGEN_STRONG_INLINE Packet4f pcast<Packet4f, Packet4f>(const Packet4f& a) {
27 return a;
28}
29template <>
30EIGEN_STRONG_INLINE Packet2f pcast<Packet2f, Packet2f>(const Packet2f& a) {
31 return a;
32}
33
34template <>
35struct type_casting_traits<float, numext::int64_t> {
36 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
37};
38template <>
39struct type_casting_traits<float, numext::uint64_t> {
40 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
41};
42// If float64 exists, first convert to that to keep as much precision as possible.
43#if EIGEN_ARCH_ARM64
44template <>
45EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
46 // Discard second half of input.
47 return vcvtq_s64_f64(vcvt_f64_f32(vget_low_f32(a)));
48}
49template <>
50EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
51 // Discard second half of input.
52 return vcvtq_u64_f64(vcvt_f64_f32(vget_low_f32(a)));
53}
54#else
55template <>
56EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
57 // Discard second half of input.
58 return vmovl_s32(vget_low_s32(vcvtq_s32_f32(a)));
59}
60template <>
61EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
62 // Discard second half of input.
63 return vmovl_u32(vget_low_u32(vcvtq_u32_f32(a)));
64}
65#endif // EIGEN_ARCH_ARM64
66
67template <>
68struct type_casting_traits<float, numext::int32_t> {
69 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
70};
71template <>
72EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
73 return vcvtq_s32_f32(a);
74}
75template <>
76EIGEN_STRONG_INLINE Packet2i pcast<Packet2f, Packet2i>(const Packet2f& a) {
77 return vcvt_s32_f32(a);
78}
79
80template <>
81struct type_casting_traits<float, numext::uint32_t> {
82 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
83};
84template <>
85EIGEN_STRONG_INLINE Packet4ui pcast<Packet4f, Packet4ui>(const Packet4f& a) {
86 return vcvtq_u32_f32(a);
87}
88template <>
89EIGEN_STRONG_INLINE Packet2ui pcast<Packet2f, Packet2ui>(const Packet2f& a) {
90 return vcvt_u32_f32(a);
91}
92
93template <>
94struct type_casting_traits<float, numext::int16_t> {
95 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
96};
97template <>
98EIGEN_STRONG_INLINE Packet8s pcast<Packet4f, Packet8s>(const Packet4f& a, const Packet4f& b) {
99 return vcombine_s16(vmovn_s32(vcvtq_s32_f32(a)), vmovn_s32(vcvtq_s32_f32(b)));
100}
101template <>
102EIGEN_STRONG_INLINE Packet4s pcast<Packet2f, Packet4s>(const Packet2f& a, const Packet2f& b) {
103 return vmovn_s32(vcombine_s32(vcvt_s32_f32(a), vcvt_s32_f32(b)));
104}
105
106template <>
107struct type_casting_traits<float, numext::uint16_t> {
108 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
109};
110template <>
111EIGEN_STRONG_INLINE Packet8us pcast<Packet4f, Packet8us>(const Packet4f& a, const Packet4f& b) {
112 return vcombine_u16(vmovn_u32(vcvtq_u32_f32(a)), vmovn_u32(vcvtq_u32_f32(b)));
113}
114template <>
115EIGEN_STRONG_INLINE Packet4us pcast<Packet2f, Packet4us>(const Packet2f& a, const Packet2f& b) {
116 return vmovn_u32(vcombine_u32(vcvt_u32_f32(a), vcvt_u32_f32(b)));
117}
118
119template <>
120struct type_casting_traits<float, numext::int8_t> {
121 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
122};
123template <>
124EIGEN_STRONG_INLINE Packet16c pcast<Packet4f, Packet16c>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
125 const Packet4f& d) {
126 const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
127 const int16x8_t cd_s16 = pcast<Packet4f, Packet8s>(c, d);
128 return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
129}
130template <>
131EIGEN_STRONG_INLINE Packet8c pcast<Packet2f, Packet8c>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
132 const Packet2f& d) {
133 const int16x4_t ab_s16 = pcast<Packet2f, Packet4s>(a, b);
134 const int16x4_t cd_s16 = pcast<Packet2f, Packet4s>(c, d);
135 return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
136}
137
138template <>
139struct type_casting_traits<float, numext::uint8_t> {
140 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
141};
142template <>
143EIGEN_STRONG_INLINE Packet16uc pcast<Packet4f, Packet16uc>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
144 const Packet4f& d) {
145 const uint16x8_t ab_u16 = pcast<Packet4f, Packet8us>(a, b);
146 const uint16x8_t cd_u16 = pcast<Packet4f, Packet8us>(c, d);
147 return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
148}
149template <>
150EIGEN_STRONG_INLINE Packet8uc pcast<Packet2f, Packet8uc>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
151 const Packet2f& d) {
152 const uint16x4_t ab_u16 = pcast<Packet2f, Packet4us>(a, b);
153 const uint16x4_t cd_u16 = pcast<Packet2f, Packet4us>(c, d);
154 return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
155}
156
157//==============================================================================
158// pcast, SrcType = int8_t
159//==============================================================================
160template <>
161struct type_casting_traits<numext::int8_t, float> {
162 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
163};
164template <>
165EIGEN_STRONG_INLINE Packet4f pcast<Packet16c, Packet4f>(const Packet16c& a) {
166 // Discard all but first 4 bytes.
167 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))));
168}
169template <>
170EIGEN_STRONG_INLINE Packet2f pcast<Packet8c, Packet2f>(const Packet8c& a) {
171 // Discard all but first 2 bytes.
172 return vcvt_f32_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a)))));
173}
174
175template <>
176struct type_casting_traits<numext::int8_t, numext::int64_t> {
177 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
178};
179template <>
180EIGEN_STRONG_INLINE Packet2l pcast<Packet16c, Packet2l>(const Packet16c& a) {
181 // Discard all but first two bytes.
182 return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))))));
183}
184
185template <>
186struct type_casting_traits<numext::int8_t, numext::uint64_t> {
187 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
188};
189template <>
190EIGEN_STRONG_INLINE Packet2ul pcast<Packet16c, Packet2ul>(const Packet16c& a) {
191 return vreinterpretq_u64_s64(pcast<Packet16c, Packet2l>(a));
192}
193
194template <>
195struct type_casting_traits<numext::int8_t, numext::int32_t> {
196 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
197};
198template <>
199EIGEN_STRONG_INLINE Packet4i pcast<Packet16c, Packet4i>(const Packet16c& a) {
200 // Discard all but first 4 bytes.
201 return vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))));
202}
203template <>
204EIGEN_STRONG_INLINE Packet2i pcast<Packet8c, Packet2i>(const Packet8c& a) {
205 // Discard all but first 2 bytes.
206 return vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a))));
207}
208
209template <>
210struct type_casting_traits<numext::int8_t, numext::uint32_t> {
211 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
212};
213template <>
214EIGEN_STRONG_INLINE Packet4ui pcast<Packet16c, Packet4ui>(const Packet16c& a) {
215 return vreinterpretq_u32_s32(pcast<Packet16c, Packet4i>(a));
216}
217template <>
218EIGEN_STRONG_INLINE Packet2ui pcast<Packet8c, Packet2ui>(const Packet8c& a) {
219 return vreinterpret_u32_s32(pcast<Packet8c, Packet2i>(a));
220}
221
222template <>
223struct type_casting_traits<numext::int8_t, numext::int16_t> {
224 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
225};
226template <>
227EIGEN_STRONG_INLINE Packet8s pcast<Packet16c, Packet8s>(const Packet16c& a) {
228 // Discard second half of input.
229 return vmovl_s8(vget_low_s8(a));
230}
231template <>
232EIGEN_STRONG_INLINE Packet4s pcast<Packet8c, Packet4s>(const Packet8c& a) {
233 // Discard second half of input.
234 return vget_low_s16(vmovl_s8(a));
235}
236
237template <>
238struct type_casting_traits<numext::int8_t, numext::uint16_t> {
239 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
240};
241template <>
242EIGEN_STRONG_INLINE Packet8us pcast<Packet16c, Packet8us>(const Packet16c& a) {
243 return vreinterpretq_u16_s16(pcast<Packet16c, Packet8s>(a));
244}
245template <>
246EIGEN_STRONG_INLINE Packet4us pcast<Packet8c, Packet4us>(const Packet8c& a) {
247 return vreinterpret_u16_s16(pcast<Packet8c, Packet4s>(a));
248}
249
250template <>
251struct type_casting_traits<numext::int8_t, numext::int8_t> {
252 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
253};
254template <>
255EIGEN_STRONG_INLINE Packet16c pcast<Packet16c, Packet16c>(const Packet16c& a) {
256 return a;
257}
258template <>
259EIGEN_STRONG_INLINE Packet8c pcast<Packet8c, Packet8c>(const Packet8c& a) {
260 return a;
261}
262template <>
263EIGEN_STRONG_INLINE Packet4c pcast<Packet4c, Packet4c>(const Packet4c& a) {
264 return a;
265}
266
267template <>
268struct type_casting_traits<numext::int8_t, numext::uint8_t> {
269 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
270};
271template <>
272EIGEN_STRONG_INLINE Packet16uc pcast<Packet16c, Packet16uc>(const Packet16c& a) {
273 return vreinterpretq_u8_s8(a);
274}
275template <>
276EIGEN_STRONG_INLINE Packet8uc pcast<Packet8c, Packet8uc>(const Packet8c& a) {
277 return vreinterpret_u8_s8(a);
278}
279template <>
280EIGEN_STRONG_INLINE Packet4uc pcast<Packet4c, Packet4uc>(const Packet4c& a) {
281 return static_cast<Packet4uc>(a);
282}
283
284//==============================================================================
285// pcast, SrcType = uint8_t
286//==============================================================================
287template <>
288struct type_casting_traits<numext::uint8_t, float> {
289 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
290};
291template <>
292EIGEN_STRONG_INLINE Packet4f pcast<Packet16uc, Packet4f>(const Packet16uc& a) {
293 // Discard all but first 4 bytes.
294 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a)))));
295}
296template <>
297EIGEN_STRONG_INLINE Packet2f pcast<Packet8uc, Packet2f>(const Packet8uc& a) {
298 // Discard all but first 2 bytes.
299 return vcvt_f32_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a)))));
300}
301
302template <>
303struct type_casting_traits<numext::uint8_t, numext::uint64_t> {
304 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
305};
306template <>
307EIGEN_STRONG_INLINE Packet2ul pcast<Packet16uc, Packet2ul>(const Packet16uc& a) {
308 // Discard all but first two bytes.
309 return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))));
310}
311
312template <>
313struct type_casting_traits<numext::uint8_t, numext::int64_t> {
314 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
315};
316template <>
317EIGEN_STRONG_INLINE Packet2l pcast<Packet16uc, Packet2l>(const Packet16uc& a) {
318 return vreinterpretq_s64_u64(pcast<Packet16uc, Packet2ul>(a));
319}
320
321template <>
322struct type_casting_traits<numext::uint8_t, numext::uint32_t> {
323 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
324};
325template <>
326EIGEN_STRONG_INLINE Packet4ui pcast<Packet16uc, Packet4ui>(const Packet16uc& a) {
327 // Discard all but first 4 bytes.
328 return vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))));
329}
330template <>
331EIGEN_STRONG_INLINE Packet2ui pcast<Packet8uc, Packet2ui>(const Packet8uc& a) {
332 // Discard all but first 2 bytes.
333 return vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a))));
334}
335
336template <>
337struct type_casting_traits<numext::uint8_t, numext::int32_t> {
338 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
339};
340template <>
341EIGEN_STRONG_INLINE Packet4i pcast<Packet16uc, Packet4i>(const Packet16uc& a) {
342 return vreinterpretq_s32_u32(pcast<Packet16uc, Packet4ui>(a));
343}
344template <>
345EIGEN_STRONG_INLINE Packet2i pcast<Packet8uc, Packet2i>(const Packet8uc& a) {
346 return vreinterpret_s32_u32(pcast<Packet8uc, Packet2ui>(a));
347}
348
349template <>
350struct type_casting_traits<numext::uint8_t, numext::uint16_t> {
351 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
352};
353template <>
354EIGEN_STRONG_INLINE Packet8us pcast<Packet16uc, Packet8us>(const Packet16uc& a) {
355 // Discard second half of input.
356 return vmovl_u8(vget_low_u8(a));
357}
358template <>
359EIGEN_STRONG_INLINE Packet4us pcast<Packet8uc, Packet4us>(const Packet8uc& a) {
360 // Discard second half of input.
361 return vget_low_u16(vmovl_u8(a));
362}
363
364template <>
365struct type_casting_traits<numext::uint8_t, numext::int16_t> {
366 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
367};
368template <>
369EIGEN_STRONG_INLINE Packet8s pcast<Packet16uc, Packet8s>(const Packet16uc& a) {
370 return vreinterpretq_s16_u16(pcast<Packet16uc, Packet8us>(a));
371}
372template <>
373EIGEN_STRONG_INLINE Packet4s pcast<Packet8uc, Packet4s>(const Packet8uc& a) {
374 return vreinterpret_s16_u16(pcast<Packet8uc, Packet4us>(a));
375}
376
377template <>
378struct type_casting_traits<numext::uint8_t, numext::uint8_t> {
379 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
380};
381template <>
382EIGEN_STRONG_INLINE Packet16uc pcast<Packet16uc, Packet16uc>(const Packet16uc& a) {
383 return a;
384}
385template <>
386EIGEN_STRONG_INLINE Packet8uc pcast<Packet8uc, Packet8uc>(const Packet8uc& a) {
387 return a;
388}
389template <>
390EIGEN_STRONG_INLINE Packet4uc pcast<Packet4uc, Packet4uc>(const Packet4uc& a) {
391 return a;
392}
393
394template <>
395struct type_casting_traits<numext::uint8_t, numext::int8_t> {
396 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
397};
398template <>
399EIGEN_STRONG_INLINE Packet16c pcast<Packet16uc, Packet16c>(const Packet16uc& a) {
400 return vreinterpretq_s8_u8(a);
401}
402template <>
403EIGEN_STRONG_INLINE Packet8c pcast<Packet8uc, Packet8c>(const Packet8uc& a) {
404 return vreinterpret_s8_u8(a);
405}
406template <>
407EIGEN_STRONG_INLINE Packet4c pcast<Packet4uc, Packet4c>(const Packet4uc& a) {
408 return static_cast<Packet4c>(a);
409}
410
411//==============================================================================
412// pcast, SrcType = int16_t
413//==============================================================================
414template <>
415struct type_casting_traits<numext::int16_t, float> {
416 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
417};
418template <>
419EIGEN_STRONG_INLINE Packet4f pcast<Packet8s, Packet4f>(const Packet8s& a) {
420 // Discard second half of input.
421 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(a)));
422}
423template <>
424EIGEN_STRONG_INLINE Packet2f pcast<Packet4s, Packet2f>(const Packet4s& a) {
425 // Discard second half of input.
426 return vcvt_f32_s32(vget_low_s32(vmovl_s16(a)));
427}
428
429template <>
430struct type_casting_traits<numext::int16_t, numext::int64_t> {
431 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
432};
433template <>
434EIGEN_STRONG_INLINE Packet2l pcast<Packet8s, Packet2l>(const Packet8s& a) {
435 // Discard all but first two values.
436 return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(a))));
437}
438
439template <>
440struct type_casting_traits<numext::int16_t, numext::uint64_t> {
441 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
442};
443template <>
444EIGEN_STRONG_INLINE Packet2ul pcast<Packet8s, Packet2ul>(const Packet8s& a) {
445 return vreinterpretq_u64_s64(pcast<Packet8s, Packet2l>(a));
446}
447
448template <>
449struct type_casting_traits<numext::int16_t, numext::int32_t> {
450 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
451};
452template <>
453EIGEN_STRONG_INLINE Packet4i pcast<Packet8s, Packet4i>(const Packet8s& a) {
454 // Discard second half of input.
455 return vmovl_s16(vget_low_s16(a));
456}
457template <>
458EIGEN_STRONG_INLINE Packet2i pcast<Packet4s, Packet2i>(const Packet4s& a) {
459 // Discard second half of input.
460 return vget_low_s32(vmovl_s16(a));
461}
462
463template <>
464struct type_casting_traits<numext::int16_t, numext::uint32_t> {
465 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
466};
467template <>
468EIGEN_STRONG_INLINE Packet4ui pcast<Packet8s, Packet4ui>(const Packet8s& a) {
469 return vreinterpretq_u32_s32(pcast<Packet8s, Packet4i>(a));
470}
471template <>
472EIGEN_STRONG_INLINE Packet2ui pcast<Packet4s, Packet2ui>(const Packet4s& a) {
473 return vreinterpret_u32_s32(pcast<Packet4s, Packet2i>(a));
474}
475
476template <>
477struct type_casting_traits<numext::int16_t, numext::int16_t> {
478 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
479};
480template <>
481EIGEN_STRONG_INLINE Packet8s pcast<Packet8s, Packet8s>(const Packet8s& a) {
482 return a;
483}
484template <>
485EIGEN_STRONG_INLINE Packet4s pcast<Packet4s, Packet4s>(const Packet4s& a) {
486 return a;
487}
488
489template <>
490struct type_casting_traits<numext::int16_t, numext::uint16_t> {
491 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
492};
493template <>
494EIGEN_STRONG_INLINE Packet8us pcast<Packet8s, Packet8us>(const Packet8s& a) {
495 return vreinterpretq_u16_s16(a);
496}
497template <>
498EIGEN_STRONG_INLINE Packet4us pcast<Packet4s, Packet4us>(const Packet4s& a) {
499 return vreinterpret_u16_s16(a);
500}
501
502template <>
503struct type_casting_traits<numext::int16_t, numext::int8_t> {
504 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
505};
506template <>
507EIGEN_STRONG_INLINE Packet16c pcast<Packet8s, Packet16c>(const Packet8s& a, const Packet8s& b) {
508 return vcombine_s8(vmovn_s16(a), vmovn_s16(b));
509}
510template <>
511EIGEN_STRONG_INLINE Packet8c pcast<Packet4s, Packet8c>(const Packet4s& a, const Packet4s& b) {
512 return vmovn_s16(vcombine_s16(a, b));
513}
514
515template <>
516struct type_casting_traits<numext::int16_t, numext::uint8_t> {
517 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
518};
519template <>
520EIGEN_STRONG_INLINE Packet16uc pcast<Packet8s, Packet16uc>(const Packet8s& a, const Packet8s& b) {
521 return vcombine_u8(vmovn_u16(vreinterpretq_u16_s16(a)), vmovn_u16(vreinterpretq_u16_s16(b)));
522}
523template <>
524EIGEN_STRONG_INLINE Packet8uc pcast<Packet4s, Packet8uc>(const Packet4s& a, const Packet4s& b) {
525 return vmovn_u16(vcombine_u16(vreinterpret_u16_s16(a), vreinterpret_u16_s16(b)));
526}
527
528//==============================================================================
529// pcast, SrcType = uint16_t
530//==============================================================================
531template <>
532struct type_casting_traits<numext::uint16_t, float> {
533 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
534};
535template <>
536EIGEN_STRONG_INLINE Packet4f pcast<Packet8us, Packet4f>(const Packet8us& a) {
537 // Discard second half of input.
538 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(a)));
539}
540template <>
541EIGEN_STRONG_INLINE Packet2f pcast<Packet4us, Packet2f>(const Packet4us& a) {
542 // Discard second half of input.
543 return vcvt_f32_u32(vget_low_u32(vmovl_u16(a)));
544}
545
546template <>
547struct type_casting_traits<numext::uint16_t, numext::uint64_t> {
548 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
549};
550template <>
551EIGEN_STRONG_INLINE Packet2ul pcast<Packet8us, Packet2ul>(const Packet8us& a) {
552 // Discard all but first two values.
553 return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(a))));
554}
555
556template <>
557struct type_casting_traits<numext::uint16_t, numext::int64_t> {
558 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
559};
560template <>
561EIGEN_STRONG_INLINE Packet2l pcast<Packet8us, Packet2l>(const Packet8us& a) {
562 return vreinterpretq_s64_u64(pcast<Packet8us, Packet2ul>(a));
563}
564
565template <>
566struct type_casting_traits<numext::uint16_t, numext::uint32_t> {
567 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
568};
569template <>
570EIGEN_STRONG_INLINE Packet4ui pcast<Packet8us, Packet4ui>(const Packet8us& a) {
571 // Discard second half of input.
572 return vmovl_u16(vget_low_u16(a));
573}
574template <>
575EIGEN_STRONG_INLINE Packet2ui pcast<Packet4us, Packet2ui>(const Packet4us& a) {
576 // Discard second half of input.
577 return vget_low_u32(vmovl_u16(a));
578}
579
580template <>
581struct type_casting_traits<numext::uint16_t, numext::int32_t> {
582 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
583};
584template <>
585EIGEN_STRONG_INLINE Packet4i pcast<Packet8us, Packet4i>(const Packet8us& a) {
586 return vreinterpretq_s32_u32(pcast<Packet8us, Packet4ui>(a));
587}
588template <>
589EIGEN_STRONG_INLINE Packet2i pcast<Packet4us, Packet2i>(const Packet4us& a) {
590 return vreinterpret_s32_u32(pcast<Packet4us, Packet2ui>(a));
591}
592
593template <>
594struct type_casting_traits<numext::uint16_t, numext::uint16_t> {
595 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
596};
597template <>
598EIGEN_STRONG_INLINE Packet8us pcast<Packet8us, Packet8us>(const Packet8us& a) {
599 return a;
600}
601template <>
602EIGEN_STRONG_INLINE Packet4us pcast<Packet4us, Packet4us>(const Packet4us& a) {
603 return a;
604}
605
606template <>
607struct type_casting_traits<numext::uint16_t, numext::int16_t> {
608 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
609};
610template <>
611EIGEN_STRONG_INLINE Packet8s pcast<Packet8us, Packet8s>(const Packet8us& a) {
612 return vreinterpretq_s16_u16(a);
613}
614template <>
615EIGEN_STRONG_INLINE Packet4s pcast<Packet4us, Packet4s>(const Packet4us& a) {
616 return vreinterpret_s16_u16(a);
617}
618
619template <>
620struct type_casting_traits<numext::uint16_t, numext::uint8_t> {
621 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
622};
623template <>
624EIGEN_STRONG_INLINE Packet16uc pcast<Packet8us, Packet16uc>(const Packet8us& a, const Packet8us& b) {
625 return vcombine_u8(vmovn_u16(a), vmovn_u16(b));
626}
627template <>
628EIGEN_STRONG_INLINE Packet8uc pcast<Packet4us, Packet8uc>(const Packet4us& a, const Packet4us& b) {
629 return vmovn_u16(vcombine_u16(a, b));
630}
631
632template <>
633struct type_casting_traits<numext::uint16_t, numext::int8_t> {
634 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
635};
636template <>
637EIGEN_STRONG_INLINE Packet16c pcast<Packet8us, Packet16c>(const Packet8us& a, const Packet8us& b) {
638 return vreinterpretq_s8_u8(pcast<Packet8us, Packet16uc>(a, b));
639}
640template <>
641EIGEN_STRONG_INLINE Packet8c pcast<Packet4us, Packet8c>(const Packet4us& a, const Packet4us& b) {
642 return vreinterpret_s8_u8(pcast<Packet4us, Packet8uc>(a, b));
643}
644
645//==============================================================================
646// pcast, SrcType = int32_t
647//==============================================================================
648template <>
649struct type_casting_traits<numext::int32_t, float> {
650 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
651};
652template <>
653EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) {
654 return vcvtq_f32_s32(a);
655}
656template <>
657EIGEN_STRONG_INLINE Packet2f pcast<Packet2i, Packet2f>(const Packet2i& a) {
658 return vcvt_f32_s32(a);
659}
660
661template <>
662struct type_casting_traits<numext::int32_t, numext::int64_t> {
663 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
664};
665template <>
666EIGEN_STRONG_INLINE Packet2l pcast<Packet4i, Packet2l>(const Packet4i& a) {
667 // Discard second half of input.
668 return vmovl_s32(vget_low_s32(a));
669}
670
671template <>
672struct type_casting_traits<numext::int32_t, numext::uint64_t> {
673 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
674};
675template <>
676EIGEN_STRONG_INLINE Packet2ul pcast<Packet4i, Packet2ul>(const Packet4i& a) {
677 return vreinterpretq_u64_s64(pcast<Packet4i, Packet2l>(a));
678}
679
680template <>
681struct type_casting_traits<numext::int32_t, numext::int32_t> {
682 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
683};
684template <>
685EIGEN_STRONG_INLINE Packet4i pcast<Packet4i, Packet4i>(const Packet4i& a) {
686 return a;
687}
688template <>
689EIGEN_STRONG_INLINE Packet2i pcast<Packet2i, Packet2i>(const Packet2i& a) {
690 return a;
691}
692
693template <>
694struct type_casting_traits<numext::int32_t, numext::uint32_t> {
695 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
696};
697template <>
698EIGEN_STRONG_INLINE Packet4ui pcast<Packet4i, Packet4ui>(const Packet4i& a) {
699 return vreinterpretq_u32_s32(a);
700}
701template <>
702EIGEN_STRONG_INLINE Packet2ui pcast<Packet2i, Packet2ui>(const Packet2i& a) {
703 return vreinterpret_u32_s32(a);
704}
705
706template <>
707struct type_casting_traits<numext::int32_t, numext::int16_t> {
708 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
709};
710template <>
711EIGEN_STRONG_INLINE Packet8s pcast<Packet4i, Packet8s>(const Packet4i& a, const Packet4i& b) {
712 return vcombine_s16(vmovn_s32(a), vmovn_s32(b));
713}
714template <>
715EIGEN_STRONG_INLINE Packet4s pcast<Packet2i, Packet4s>(const Packet2i& a, const Packet2i& b) {
716 return vmovn_s32(vcombine_s32(a, b));
717}
718
719template <>
720struct type_casting_traits<numext::int32_t, numext::uint16_t> {
721 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
722};
723template <>
724EIGEN_STRONG_INLINE Packet8us pcast<Packet4i, Packet8us>(const Packet4i& a, const Packet4i& b) {
725 return vcombine_u16(vmovn_u32(vreinterpretq_u32_s32(a)), vmovn_u32(vreinterpretq_u32_s32(b)));
726}
727template <>
728EIGEN_STRONG_INLINE Packet4us pcast<Packet2i, Packet4us>(const Packet2i& a, const Packet2i& b) {
729 return vmovn_u32(vreinterpretq_u32_s32(vcombine_s32(a, b)));
730}
731
732template <>
733struct type_casting_traits<numext::int32_t, numext::int8_t> {
734 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
735};
736template <>
737EIGEN_STRONG_INLINE Packet16c pcast<Packet4i, Packet16c>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
738 const Packet4i& d) {
739 const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
740 const int16x8_t cd_s16 = pcast<Packet4i, Packet8s>(c, d);
741 return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
742}
743template <>
744EIGEN_STRONG_INLINE Packet8c pcast<Packet2i, Packet8c>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
745 const Packet2i& d) {
746 const int16x4_t ab_s16 = vmovn_s32(vcombine_s32(a, b));
747 const int16x4_t cd_s16 = vmovn_s32(vcombine_s32(c, d));
748 return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
749}
750
751template <>
752struct type_casting_traits<numext::int32_t, numext::uint8_t> {
753 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
754};
755template <>
756EIGEN_STRONG_INLINE Packet16uc pcast<Packet4i, Packet16uc>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
757 const Packet4i& d) {
758 const uint16x8_t ab_u16 = pcast<Packet4i, Packet8us>(a, b);
759 const uint16x8_t cd_u16 = pcast<Packet4i, Packet8us>(c, d);
760 return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
761}
762template <>
763EIGEN_STRONG_INLINE Packet8uc pcast<Packet2i, Packet8uc>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
764 const Packet2i& d) {
765 const uint16x4_t ab_u16 = pcast<Packet2i, Packet4us>(a, b);
766 const uint16x4_t cd_u16 = pcast<Packet2i, Packet4us>(c, d);
767 return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
768}
769
770//==============================================================================
771// pcast, SrcType = uint32_t
772//==============================================================================
773template <>
774struct type_casting_traits<numext::uint32_t, float> {
775 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
776};
777template <>
778EIGEN_STRONG_INLINE Packet4f pcast<Packet4ui, Packet4f>(const Packet4ui& a) {
779 return vcvtq_f32_u32(a);
780}
781template <>
782EIGEN_STRONG_INLINE Packet2f pcast<Packet2ui, Packet2f>(const Packet2ui& a) {
783 return vcvt_f32_u32(a);
784}
785
786template <>
787struct type_casting_traits<numext::uint32_t, numext::uint64_t> {
788 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
789};
790template <>
791EIGEN_STRONG_INLINE Packet2ul pcast<Packet4ui, Packet2ul>(const Packet4ui& a) {
792 // Discard second half of input.
793 return vmovl_u32(vget_low_u32(a));
794}
795
796template <>
797struct type_casting_traits<numext::uint32_t, numext::int64_t> {
798 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
799};
800template <>
801EIGEN_STRONG_INLINE Packet2l pcast<Packet4ui, Packet2l>(const Packet4ui& a) {
802 return vreinterpretq_s64_u64(pcast<Packet4ui, Packet2ul>(a));
803}
804
805template <>
806struct type_casting_traits<numext::uint32_t, numext::uint32_t> {
807 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
808};
809template <>
810EIGEN_STRONG_INLINE Packet4ui pcast<Packet4ui, Packet4ui>(const Packet4ui& a) {
811 return a;
812}
813template <>
814EIGEN_STRONG_INLINE Packet2ui pcast<Packet2ui, Packet2ui>(const Packet2ui& a) {
815 return a;
816}
817
818template <>
819struct type_casting_traits<numext::uint32_t, numext::int32_t> {
820 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
821};
822template <>
823EIGEN_STRONG_INLINE Packet4i pcast<Packet4ui, Packet4i>(const Packet4ui& a) {
824 return vreinterpretq_s32_u32(a);
825}
826template <>
827EIGEN_STRONG_INLINE Packet2i pcast<Packet2ui, Packet2i>(const Packet2ui& a) {
828 return vreinterpret_s32_u32(a);
829}
830
831template <>
832struct type_casting_traits<numext::uint32_t, numext::uint16_t> {
833 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
834};
835template <>
836EIGEN_STRONG_INLINE Packet8us pcast<Packet4ui, Packet8us>(const Packet4ui& a, const Packet4ui& b) {
837 return vcombine_u16(vmovn_u32(a), vmovn_u32(b));
838}
839template <>
840EIGEN_STRONG_INLINE Packet4us pcast<Packet2ui, Packet4us>(const Packet2ui& a, const Packet2ui& b) {
841 return vmovn_u32(vcombine_u32(a, b));
842}
843
844template <>
845struct type_casting_traits<numext::uint32_t, numext::int16_t> {
846 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
847};
848template <>
849EIGEN_STRONG_INLINE Packet8s pcast<Packet4ui, Packet8s>(const Packet4ui& a, const Packet4ui& b) {
850 return vreinterpretq_s16_u16(pcast<Packet4ui, Packet8us>(a, b));
851}
852template <>
853EIGEN_STRONG_INLINE Packet4s pcast<Packet2ui, Packet4s>(const Packet2ui& a, const Packet2ui& b) {
854 return vreinterpret_s16_u16(pcast<Packet2ui, Packet4us>(a, b));
855}
856
857template <>
858struct type_casting_traits<numext::uint32_t, numext::uint8_t> {
859 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
860};
861template <>
862EIGEN_STRONG_INLINE Packet16uc pcast<Packet4ui, Packet16uc>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
863 const Packet4ui& d) {
864 const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
865 const uint16x8_t cd_u16 = vcombine_u16(vmovn_u32(c), vmovn_u32(d));
866 return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
867}
868template <>
869EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ui, Packet8uc>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
870 const Packet2ui& d) {
871 const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(a, b));
872 const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(c, d));
873 return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
874}
875
876template <>
877struct type_casting_traits<numext::uint32_t, numext::int8_t> {
878 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
879};
880template <>
881EIGEN_STRONG_INLINE Packet16c pcast<Packet4ui, Packet16c>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
882 const Packet4ui& d) {
883 return vreinterpretq_s8_u8(pcast<Packet4ui, Packet16uc>(a, b, c, d));
884}
885template <>
886EIGEN_STRONG_INLINE Packet8c pcast<Packet2ui, Packet8c>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
887 const Packet2ui& d) {
888 return vreinterpret_s8_u8(pcast<Packet2ui, Packet8uc>(a, b, c, d));
889}
890
891//==============================================================================
892// pcast, SrcType = int64_t
893//==============================================================================
894template <>
895struct type_casting_traits<numext::int64_t, float> {
896 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
897};
898template <>
899EIGEN_STRONG_INLINE Packet4f pcast<Packet2l, Packet4f>(const Packet2l& a, const Packet2l& b) {
900 return vcvtq_f32_s32(vcombine_s32(vmovn_s64(a), vmovn_s64(b)));
901}
902
903template <>
904struct type_casting_traits<numext::int64_t, numext::int64_t> {
905 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
906};
907template <>
908EIGEN_STRONG_INLINE Packet2l pcast<Packet2l, Packet2l>(const Packet2l& a) {
909 return a;
910}
911
912template <>
913struct type_casting_traits<numext::int64_t, numext::uint64_t> {
914 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
915};
916template <>
917EIGEN_STRONG_INLINE Packet2ul pcast<Packet2l, Packet2ul>(const Packet2l& a) {
918 return vreinterpretq_u64_s64(a);
919}
920
921template <>
922struct type_casting_traits<numext::int64_t, numext::int32_t> {
923 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
924};
925template <>
926EIGEN_STRONG_INLINE Packet4i pcast<Packet2l, Packet4i>(const Packet2l& a, const Packet2l& b) {
927 return vcombine_s32(vmovn_s64(a), vmovn_s64(b));
928}
929
930template <>
931struct type_casting_traits<numext::int64_t, numext::uint32_t> {
932 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
933};
934template <>
935EIGEN_STRONG_INLINE Packet4ui pcast<Packet2l, Packet4ui>(const Packet2l& a, const Packet2l& b) {
936 return vcombine_u32(vmovn_u64(vreinterpretq_u64_s64(a)), vmovn_u64(vreinterpretq_u64_s64(b)));
937}
938
939template <>
940struct type_casting_traits<numext::int64_t, numext::int16_t> {
941 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
942};
943template <>
944EIGEN_STRONG_INLINE Packet8s pcast<Packet2l, Packet8s>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
945 const Packet2l& d) {
946 const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
947 const int32x4_t cd_s32 = pcast<Packet2l, Packet4i>(c, d);
948 return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
949}
950
951template <>
952struct type_casting_traits<numext::int64_t, numext::uint16_t> {
953 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
954};
955template <>
956EIGEN_STRONG_INLINE Packet8us pcast<Packet2l, Packet8us>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
957 const Packet2l& d) {
958 const uint32x4_t ab_u32 = pcast<Packet2l, Packet4ui>(a, b);
959 const uint32x4_t cd_u32 = pcast<Packet2l, Packet4ui>(c, d);
960 return vcombine_u16(vmovn_u32(ab_u32), vmovn_u32(cd_u32));
961}
962
963template <>
964struct type_casting_traits<numext::int64_t, numext::int8_t> {
965 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
966};
967template <>
968EIGEN_STRONG_INLINE Packet16c pcast<Packet2l, Packet16c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
969 const Packet2l& d, const Packet2l& e, const Packet2l& f,
970 const Packet2l& g, const Packet2l& h) {
971 const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
972 const int16x8_t efgh_s16 = pcast<Packet2l, Packet8s>(e, f, g, h);
973 return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
974}
975
976template <>
977struct type_casting_traits<numext::int64_t, numext::uint8_t> {
978 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
979};
980template <>
981EIGEN_STRONG_INLINE Packet16uc pcast<Packet2l, Packet16uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
982 const Packet2l& d, const Packet2l& e, const Packet2l& f,
983 const Packet2l& g, const Packet2l& h) {
984 const uint16x8_t abcd_u16 = pcast<Packet2l, Packet8us>(a, b, c, d);
985 const uint16x8_t efgh_u16 = pcast<Packet2l, Packet8us>(e, f, g, h);
986 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
987}
988
989//==============================================================================
990// pcast, SrcType = uint64_t
991//==============================================================================
992template <>
993struct type_casting_traits<numext::uint64_t, float> {
994 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
995};
996template <>
997EIGEN_STRONG_INLINE Packet4f pcast<Packet2ul, Packet4f>(const Packet2ul& a, const Packet2ul& b) {
998 return vcvtq_f32_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
999}
1000
1001template <>
1002struct type_casting_traits<numext::uint64_t, numext::uint64_t> {
1003 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1004};
1005template <>
1006EIGEN_STRONG_INLINE Packet2ul pcast<Packet2ul, Packet2ul>(const Packet2ul& a) {
1007 return a;
1008}
1009
1010template <>
1011struct type_casting_traits<numext::uint64_t, numext::int64_t> {
1012 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1013};
1014template <>
1015EIGEN_STRONG_INLINE Packet2l pcast<Packet2ul, Packet2l>(const Packet2ul& a) {
1016 return vreinterpretq_s64_u64(a);
1017}
1018
1019template <>
1020struct type_casting_traits<numext::uint64_t, numext::uint32_t> {
1021 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1022};
1023template <>
1024EIGEN_STRONG_INLINE Packet4ui pcast<Packet2ul, Packet4ui>(const Packet2ul& a, const Packet2ul& b) {
1025 return vcombine_u32(vmovn_u64(a), vmovn_u64(b));
1026}
1027
1028template <>
1029struct type_casting_traits<numext::uint64_t, numext::int32_t> {
1030 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1031};
1032template <>
1033EIGEN_STRONG_INLINE Packet4i pcast<Packet2ul, Packet4i>(const Packet2ul& a, const Packet2ul& b) {
1034 return vreinterpretq_s32_u32(pcast<Packet2ul, Packet4ui>(a, b));
1035}
1036
1037template <>
1038struct type_casting_traits<numext::uint64_t, numext::uint16_t> {
1039 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1040};
1041template <>
1042EIGEN_STRONG_INLINE Packet8us pcast<Packet2ul, Packet8us>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1043 const Packet2ul& d) {
1044 const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1045 const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(vmovn_u64(c), vmovn_u64(d)));
1046 return vcombine_u16(ab_u16, cd_u16);
1047}
1048
1049template <>
1050struct type_casting_traits<numext::uint64_t, numext::int16_t> {
1051 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1052};
1053template <>
1054EIGEN_STRONG_INLINE Packet8s pcast<Packet2ul, Packet8s>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1055 const Packet2ul& d) {
1056 return vreinterpretq_s16_u16(pcast<Packet2ul, Packet8us>(a, b, c, d));
1057}
1058
1059template <>
1060struct type_casting_traits<numext::uint64_t, numext::uint8_t> {
1061 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1062};
1063template <>
1064EIGEN_STRONG_INLINE Packet16uc pcast<Packet2ul, Packet16uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1065 const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1066 const Packet2ul& g, const Packet2ul& h) {
1067 const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1068 const uint16x8_t efgh_u16 = pcast<Packet2ul, Packet8us>(e, f, g, h);
1069 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1070}
1071
1072template <>
1073struct type_casting_traits<numext::uint64_t, numext::int8_t> {
1074 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1075};
1076template <>
1077EIGEN_STRONG_INLINE Packet16c pcast<Packet2ul, Packet16c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1078 const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1079 const Packet2ul& g, const Packet2ul& h) {
1080 return vreinterpretq_s8_u8(pcast<Packet2ul, Packet16uc>(a, b, c, d, e, f, g, h));
1081}
1082
1083//==============================================================================
1084// preinterpret
1085//==============================================================================
1086template <>
1087EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2i>(const Packet2i& a) {
1088 return vreinterpret_f32_s32(a);
1089}
1090template <>
1091EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2ui>(const Packet2ui& a) {
1092 return vreinterpret_f32_u32(a);
1093}
1094template <>
1095EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4i>(const Packet4i& a) {
1096 return vreinterpretq_f32_s32(a);
1097}
1098template <>
1099EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4ui>(const Packet4ui& a) {
1100 return vreinterpretq_f32_u32(a);
1101}
1102
1103template <>
1104EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet4uc>(const Packet4uc& a) {
1105 return static_cast<Packet4c>(a);
1106}
1107template <>
1108EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet8uc>(const Packet8uc& a) {
1109 return vreinterpret_s8_u8(a);
1110}
1111template <>
1112EIGEN_STRONG_INLINE Packet16c preinterpret<Packet16c, Packet16uc>(const Packet16uc& a) {
1113 return vreinterpretq_s8_u8(a);
1114}
1115
1116template <>
1117EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet4c>(const Packet4c& a) {
1118 return static_cast<Packet4uc>(a);
1119}
1120template <>
1121EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet8c>(const Packet8c& a) {
1122 return vreinterpret_u8_s8(a);
1123}
1124template <>
1125EIGEN_STRONG_INLINE Packet16uc preinterpret<Packet16uc, Packet16c>(const Packet16c& a) {
1126 return vreinterpretq_u8_s8(a);
1127}
1128
1129template <>
1130EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet4us>(const Packet4us& a) {
1131 return vreinterpret_s16_u16(a);
1132}
1133template <>
1134EIGEN_STRONG_INLINE Packet8s preinterpret<Packet8s, Packet8us>(const Packet8us& a) {
1135 return vreinterpretq_s16_u16(a);
1136}
1137
1138template <>
1139EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet4s>(const Packet4s& a) {
1140 return vreinterpret_u16_s16(a);
1141}
1142template <>
1143EIGEN_STRONG_INLINE Packet8us preinterpret<Packet8us, Packet8s>(const Packet8s& a) {
1144 return vreinterpretq_u16_s16(a);
1145}
1146
1147template <>
1148EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2f>(const Packet2f& a) {
1149 return vreinterpret_s32_f32(a);
1150}
1151template <>
1152EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2ui>(const Packet2ui& a) {
1153 return vreinterpret_s32_u32(a);
1154}
1155template <>
1156EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4f>(const Packet4f& a) {
1157 return vreinterpretq_s32_f32(a);
1158}
1159template <>
1160EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4ui>(const Packet4ui& a) {
1161 return vreinterpretq_s32_u32(a);
1162}
1163
1164template <>
1165EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2f>(const Packet2f& a) {
1166 return vreinterpret_u32_f32(a);
1167}
1168template <>
1169EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2i>(const Packet2i& a) {
1170 return vreinterpret_u32_s32(a);
1171}
1172template <>
1173EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4f>(const Packet4f& a) {
1174 return vreinterpretq_u32_f32(a);
1175}
1176template <>
1177EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4i>(const Packet4i& a) {
1178 return vreinterpretq_u32_s32(a);
1179}
1180
1181template <>
1182EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2ul>(const Packet2ul& a) {
1183 return vreinterpretq_s64_u64(a);
1184}
1185template <>
1186EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2l>(const Packet2l& a) {
1187 return vreinterpretq_u64_s64(a);
1188}
1189
1190#if EIGEN_ARCH_ARM64
1191
1192//==============================================================================
1193// pcast/preinterpret, Double
1194//==============================================================================
1195
1196template <>
1197struct type_casting_traits<double, double> {
1198 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1199};
1200template <>
1201EIGEN_STRONG_INLINE Packet2d pcast<Packet2d, Packet2d>(const Packet2d& a) {
1202 return a;
1203}
1204
1205template <>
1206struct type_casting_traits<double, float> {
1207 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1208};
1209template <>
1210EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(const Packet2d& a, const Packet2d& b) {
1211 return vcombine_f32(vcvt_f32_f64(a), vcvt_f32_f64(b));
1212}
1213
1214template <>
1215struct type_casting_traits<double, numext::int64_t> {
1216 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1217};
1218template <>
1219EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(const Packet2d& a) {
1220 return vcvtq_s64_f64(a);
1221}
1222
1223template <>
1224struct type_casting_traits<double, numext::uint64_t> {
1225 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1226};
1227template <>
1228EIGEN_STRONG_INLINE Packet2ul pcast<Packet2d, Packet2ul>(const Packet2d& a) {
1229 return vcvtq_u64_f64(a);
1230}
1231
1232template <>
1233struct type_casting_traits<double, numext::int32_t> {
1234 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1235};
1236template <>
1237EIGEN_STRONG_INLINE Packet4i pcast<Packet2d, Packet4i>(const Packet2d& a, const Packet2d& b) {
1238 return vcombine_s32(vmovn_s64(vcvtq_s64_f64(a)), vmovn_s64(vcvtq_s64_f64(b)));
1239}
1240
1241template <>
1242struct type_casting_traits<double, numext::uint32_t> {
1243 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1244};
1245template <>
1246EIGEN_STRONG_INLINE Packet4ui pcast<Packet2d, Packet4ui>(const Packet2d& a, const Packet2d& b) {
1247 return vcombine_u32(vmovn_u64(vcvtq_u64_f64(a)), vmovn_u64(vcvtq_u64_f64(b)));
1248}
1249
1250template <>
1251struct type_casting_traits<double, numext::int16_t> {
1252 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1253};
1254template <>
1255EIGEN_STRONG_INLINE Packet8s pcast<Packet2d, Packet8s>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1256 const Packet2d& d) {
1257 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1258 const int32x4_t cd_s32 = pcast<Packet2d, Packet4i>(c, d);
1259 return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1260}
1261
1262template <>
1263struct type_casting_traits<double, numext::uint16_t> {
1264 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1265};
1266template <>
1267EIGEN_STRONG_INLINE Packet8us pcast<Packet2d, Packet8us>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1268 const Packet2d& d) {
1269 const uint32x4_t ab_u32 = pcast<Packet2d, Packet4ui>(a, b);
1270 const uint32x4_t cd_u32 = pcast<Packet2d, Packet4ui>(c, d);
1271 return vcombine_u16(vmovn_u32(ab_u32), vmovn_u32(cd_u32));
1272}
1273
1274template <>
1275struct type_casting_traits<double, numext::int8_t> {
1276 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1277};
1278template <>
1279EIGEN_STRONG_INLINE Packet16c pcast<Packet2d, Packet16c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1280 const Packet2d& d, const Packet2d& e, const Packet2d& f,
1281 const Packet2d& g, const Packet2d& h) {
1282 const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1283 const int16x8_t efgh_s16 = pcast<Packet2d, Packet8s>(e, f, g, h);
1284 return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1285}
1286
1287template <>
1288struct type_casting_traits<double, numext::uint8_t> {
1289 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1290};
1291template <>
1292EIGEN_STRONG_INLINE Packet16uc pcast<Packet2d, Packet16uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1293 const Packet2d& d, const Packet2d& e, const Packet2d& f,
1294 const Packet2d& g, const Packet2d& h) {
1295 const uint16x8_t abcd_u16 = pcast<Packet2d, Packet8us>(a, b, c, d);
1296 const uint16x8_t efgh_u16 = pcast<Packet2d, Packet8us>(e, f, g, h);
1297 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1298}
1299
1300template <>
1301struct type_casting_traits<float, double> {
1302 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1303};
1304template <>
1305EIGEN_STRONG_INLINE Packet2d pcast<Packet4f, Packet2d>(const Packet4f& a) {
1306 // Discard second-half of input.
1307 return vcvt_f64_f32(vget_low_f32(a));
1308}
1309
1310template <>
1311struct type_casting_traits<numext::int8_t, double> {
1312 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1313};
1314template <>
1315EIGEN_STRONG_INLINE Packet2d pcast<Packet16c, Packet2d>(const Packet16c& a) {
1316 // Discard all but first two values.
1317 return vcvt_f64_f32(pcast<Packet8c, Packet2f>(vget_low_s8(a)));
1318}
1319
1320template <>
1321struct type_casting_traits<numext::uint8_t, double> {
1322 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1323};
1324template <>
1325EIGEN_STRONG_INLINE Packet2d pcast<Packet16uc, Packet2d>(const Packet16uc& a) {
1326 // Discard all but first two values.
1327 return vcvt_f64_f32(pcast<Packet8uc, Packet2f>(vget_low_u8(a)));
1328}
1329
1330template <>
1331struct type_casting_traits<numext::int16_t, double> {
1332 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1333};
1334template <>
1335EIGEN_STRONG_INLINE Packet2d pcast<Packet8s, Packet2d>(const Packet8s& a) {
1336 // Discard all but first two values.
1337 return vcvt_f64_f32(pcast<Packet4s, Packet2f>(vget_low_s16(a)));
1338}
1339
1340template <>
1341struct type_casting_traits<numext::uint16_t, double> {
1342 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1343};
1344template <>
1345EIGEN_STRONG_INLINE Packet2d pcast<Packet8us, Packet2d>(const Packet8us& a) {
1346 // Discard all but first two values.
1347 return vcvt_f64_f32(pcast<Packet4us, Packet2f>(vget_low_u16(a)));
1348}
1349
1350template <>
1351struct type_casting_traits<numext::int32_t, double> {
1352 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1353};
1354template <>
1355EIGEN_STRONG_INLINE Packet2d pcast<Packet4i, Packet2d>(const Packet4i& a) {
1356 // Discard second half of input.
1357 return vcvtq_f64_s64(vmovl_s32(vget_low_s32(a)));
1358}
1359
1360template <>
1361struct type_casting_traits<numext::uint32_t, double> {
1362 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1363};
1364template <>
1365EIGEN_STRONG_INLINE Packet2d pcast<Packet4ui, Packet2d>(const Packet4ui& a) {
1366 // Discard second half of input.
1367 return vcvtq_f64_u64(vmovl_u32(vget_low_u32(a)));
1368}
1369
1370template <>
1371struct type_casting_traits<numext::int64_t, double> {
1372 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1373};
1374template <>
1375EIGEN_STRONG_INLINE Packet2d pcast<Packet2l, Packet2d>(const Packet2l& a) {
1376 return vcvtq_f64_s64(a);
1377}
1378
1379template <>
1380struct type_casting_traits<numext::uint64_t, double> {
1381 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1382};
1383template <>
1384EIGEN_STRONG_INLINE Packet2d pcast<Packet2ul, Packet2d>(const Packet2ul& a) {
1385 return vcvtq_f64_u64(a);
1386}
1387
1388template <>
1389EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2l>(const Packet2l& a) {
1390 return vreinterpretq_f64_s64(a);
1391}
1392template <>
1393EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2ul>(const Packet2ul& a) {
1394 return vreinterpretq_f64_u64(a);
1395}
1396template <>
1397EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2d>(const Packet2d& a) {
1398 return vreinterpretq_s64_f64(a);
1399}
1400template <>
1401EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2d>(const Packet2d& a) {
1402 return vreinterpretq_u64_f64(a);
1403}
1404template <>
1405EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4i>(const Packet4i& a) {
1406 return vreinterpretq_f64_s32(a);
1407}
1408template <>
1409EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet2d>(const Packet2d& a) {
1410 return vreinterpretq_s32_f64(a);
1411}
1412
1413#endif // EIGEN_ARCH_ARM64
1414
1415} // end namespace internal
1416
1417} // end namespace Eigen
1418
1419#endif // EIGEN_TYPE_CASTING_NEON_H
Namespace containing all symbols from the Eigen library.
Definition: Core:141