fork/rust-sdks

git clone https://git.julia.blue/fork/rust-sdks

Théo Monnomsoxr-sys (#438)2e4d385

main
66.2 KiB1946 linesraw
1/* https://bitbucket.org/jpommier/pffft/raw/483453d8f7661058e74aa4e7cf5c27bcd7887e7a/pffft.c
2 * with minor changes for libsoxr. */
3
4/* Copyright (c) 2013  Julien Pommier ( pommier@modartt.com )
5
6   Based on original fortran 77 code from FFTPACKv4 from NETLIB
7   (http://www.netlib.org/fftpack), authored by Dr Paul Swarztrauber
8   of NCAR, in 1985.
9
10   As confirmed by the NCAR fftpack software curators, the following
11   FFTPACKv5 license applies to FFTPACKv4 sources. My changes are
12   released under the same terms.
13
14   FFTPACK license:
15
16   http://www.cisl.ucar.edu/css/software/fftpack5/ftpk.html
17
18   Copyright (c) 2004 the University Corporation for Atmospheric
19   Research ("UCAR"). All rights reserved. Developed by NCAR's
20   Computational and Information Systems Laboratory, UCAR,
21   www.cisl.ucar.edu.
22
23   Redistribution and use of the Software in source and binary forms,
24   with or without modification, is permitted provided that the
25   following conditions are met:
26
27   - Neither the names of NCAR's Computational and Information Systems
28   Laboratory, the University Corporation for Atmospheric Research,
29   nor the names of its sponsors or contributors may be used to
30   endorse or promote products derived from this Software without
31   specific prior written permission.
32
33   - Redistributions of source code must retain the above copyright
34   notices, this list of conditions, and the disclaimer below.
35
36   - Redistributions in binary form must reproduce the above copyright
37   notice, this list of conditions, and the disclaimer below in the
38   documentation and/or other materials provided with the
39   distribution.
40
41   THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
42   EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO THE WARRANTIES OF
43   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
44   NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT
45   HOLDERS BE LIABLE FOR ANY CLAIM, INDIRECT, INCIDENTAL, SPECIAL,
46   EXEMPLARY, OR CONSEQUENTIAL DAMAGES OR OTHER LIABILITY, WHETHER IN AN
47   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
48   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
49   SOFTWARE.
50
51
52   PFFFT : a Pretty Fast FFT.
53
54   This file is largerly based on the original FFTPACK implementation, modified in
55   order to take advantage of SIMD instructions of modern CPUs.
56*/
57
58/*
59  ChangeLog:
60  - 2011/10/02, version 1: This is the very first release of this file.
61*/
62
63#include "pffft.h"
64#include <stdlib.h>
65#include <stdio.h>
66#include <math.h>
67#include <assert.h>
68
69/* detect compiler flavour */
70#if defined(_MSC_VER)
71#  define COMPILER_MSVC
72#elif defined(__GNUC__)
73#  define COMPILER_GCC
74#endif
75
76#if defined(COMPILER_GCC)
77#  define ALWAYS_INLINE(return_type) inline return_type __attribute__ ((always_inline))
78#  define NEVER_INLINE(return_type) return_type __attribute__ ((noinline))
79#  define RESTRICT __restrict
80#  define VLA_ARRAY_ON_STACK(type__, varname__, size__) type__ varname__[size__];
81#elif defined(COMPILER_MSVC)
82#  define ALWAYS_INLINE(return_type) __forceinline return_type
83#  define NEVER_INLINE(return_type) __declspec(noinline) return_type
84#  define RESTRICT __restrict
85#  define VLA_ARRAY_ON_STACK(type__, varname__, size__) type__ *varname__ = (type__*)_alloca(size__ * sizeof(type__))
86#endif
87
88
89/*
90   vector support macros: the rest of the code is independant of
91   SSE/Altivec/NEON -- adding support for other platforms with 4-element
92   vectors should be limited to these macros
93*/
94
95
96/* define PFFFT_SIMD_DISABLE if you want to use scalar code instead of simd code */
97/*#define PFFFT_SIMD_DISABLE */
98
99/*
100   Altivec support macros
101*/
102#if !defined(PFFFT_SIMD_DISABLE) && (defined(__ppc__) || defined(__ppc64__))
103typedef vector float v4sf;
104#  define SIMD_SZ 4
105#  define VZERO() ((vector float) vec_splat_u8(0))
106#  define VMUL(a,b) vec_madd(a,b, VZERO())
107#  define VADD(a,b) vec_add(a,b)
108#  define VMADD(a,b,c) vec_madd(a,b,c)
109#  define VSUB(a,b) vec_sub(a,b)
110inline v4sf ld_ps1(const float *p) { v4sf v=vec_lde(0,p); return vec_splat(vec_perm(v, v, vec_lvsl(0, p)), 0); }
111#  define LD_PS1(p) ld_ps1(&p)
112#  define INTERLEAVE2(in1, in2, out1, out2) { v4sf tmp__ = vec_mergeh(in1, in2); out2 = vec_mergel(in1, in2); out1 = tmp__; }
113#  define UNINTERLEAVE2(in1, in2, out1, out2) {                           \
114    vector unsigned char vperm1 =  (vector unsigned char)(0,1,2,3,8,9,10,11,16,17,18,19,24,25,26,27); \
115    vector unsigned char vperm2 =  (vector unsigned char)(4,5,6,7,12,13,14,15,20,21,22,23,28,29,30,31); \
116    v4sf tmp__ = vec_perm(in1, in2, vperm1); out2 = vec_perm(in1, in2, vperm2); out1 = tmp__; \
117  }
118#  define VTRANSPOSE4(x0,x1,x2,x3) {              \
119    v4sf y0 = vec_mergeh(x0, x2);               \
120    v4sf y1 = vec_mergel(x0, x2);               \
121    v4sf y2 = vec_mergeh(x1, x3);               \
122    v4sf y3 = vec_mergel(x1, x3);               \
123    x0 = vec_mergeh(y0, y2);                    \
124    x1 = vec_mergel(y0, y2);                    \
125    x2 = vec_mergeh(y1, y3);                    \
126    x3 = vec_mergel(y1, y3);                    \
127  }
128#  define VSWAPHL(a,b) vec_perm(a,b, (vector unsigned char)(16,17,18,19,20,21,22,23,8,9,10,11,12,13,14,15))
129#  define VALIGNED(ptr) ((((long)(ptr)) & 0xF) == 0)
130
131/*
132  SSE1 support macros
133*/
134#elif !defined(PFFFT_SIMD_DISABLE) && (defined(__x86_64__) || defined(_M_X64) || defined(i386) || defined(_M_IX86))
135
136#  define SIMD_SZ 4 /* 4 floats by simd vector -- this is pretty much hardcoded in the preprocess/finalize functions anyway so you will have to work if you want to enable AVX with its 256-bit vectors. */
137
138#if !PFFFT_DOUBLE
139#include <xmmintrin.h>
140typedef __m128 v4sf;
141#  define VZERO() _mm_setzero_ps()
142#  define VMUL(a,b) _mm_mul_ps(a,b)
143#  define VADD(a,b) _mm_add_ps(a,b)
144#  define VMADD(a,b,c) _mm_add_ps(_mm_mul_ps(a,b), c)
145#  define VSUB(a,b) _mm_sub_ps(a,b)
146#  define LD_PS1(p) _mm_set1_ps(p)
147#  define INTERLEAVE2(in1, in2, out1, out2) { v4sf tmp__ = _mm_unpacklo_ps(in1, in2); out2 = _mm_unpackhi_ps(in1, in2); out1 = tmp__; }
148#  define UNINTERLEAVE2(in1, in2, out1, out2) { v4sf tmp__ = _mm_shuffle_ps(in1, in2, _MM_SHUFFLE(2,0,2,0)); out2 = _mm_shuffle_ps(in1, in2, _MM_SHUFFLE(3,1,3,1)); out1 = tmp__; }
149#  define VTRANSPOSE4(x0,x1,x2,x3) _MM_TRANSPOSE4_PS(x0,x1,x2,x3)
150#  define VSWAPHL(a,b) _mm_shuffle_ps(b, a, _MM_SHUFFLE(3,2,1,0))
151#  define VALIGNED(ptr) ((((long)(ptr)) & 0xF) == 0)
152
153#else
154#include "pffft-avx.h"
155#endif
156
157/*
158  ARM NEON support macros
159*/
160#elif !defined(PFFFT_SIMD_DISABLE) && defined(__arm__)
161#  include <arm_neon.h>
162typedef float32x4_t v4sf;
163#  define SIMD_SZ 4
164#  define VZERO() vdupq_n_f32(0)
165#  define VMUL(a,b) vmulq_f32(a,b)
166#  define VADD(a,b) vaddq_f32(a,b)
167#  define VMADD(a,b,c) vmlaq_f32(c,a,b)
168#  define VSUB(a,b) vsubq_f32(a,b)
169#  define LD_PS1(p) vld1q_dup_f32(&(p))
170#  define INTERLEAVE2(in1, in2, out1, out2) { float32x4x2_t tmp__ = vzipq_f32(in1,in2); out1=tmp__.val[0]; out2=tmp__.val[1]; }
171#  define UNINTERLEAVE2(in1, in2, out1, out2) { float32x4x2_t tmp__ = vuzpq_f32(in1,in2); out1=tmp__.val[0]; out2=tmp__.val[1]; }
172#  define VTRANSPOSE4(x0,x1,x2,x3) {                                    \
173    float32x4x2_t t0_ = vzipq_f32(x0, x2);                              \
174    float32x4x2_t t1_ = vzipq_f32(x1, x3);                              \
175    float32x4x2_t u0_ = vzipq_f32(t0_.val[0], t1_.val[0]);              \
176    float32x4x2_t u1_ = vzipq_f32(t0_.val[1], t1_.val[1]);              \
177    x0 = u0_.val[0]; x1 = u0_.val[1]; x2 = u1_.val[0]; x3 = u1_.val[1]; \
178  }
179/* marginally faster version */
180/*#  define VTRANSPOSE4(x0,x1,x2,x3) { asm("vtrn.32 %q0, %q1;\n vtrn.32 %q2,%q3\n vswp %f0,%e2\n vswp %f1,%e3" : "+w"(x0), "+w"(x1), "+w"(x2), "+w"(x3)::); } */
181#  define VSWAPHL(a,b) vcombine_f32(vget_low_f32(b), vget_high_f32(a))
182#  define VALIGNED(ptr) ((((long)(ptr)) & 0x3) == 0)
183#else
184#  if !defined(PFFFT_SIMD_DISABLE)
185#    warning "building with simd disabled !\n";
186#    define PFFFT_SIMD_DISABLE /* fallback to scalar code */
187#  endif
188#endif
189
190#if PFFFT_DOUBLE
191#define float double
192#endif
193
194/* fallback mode for situations where SSE/Altivec are not available, use scalar mode instead */
195#ifdef PFFFT_SIMD_DISABLE
196typedef float v4sf;
197#  define SIMD_SZ 1
198#  define VZERO() 0.f
199#  define VMUL(a,b) ((a)*(b))
200#  define VADD(a,b) ((a)+(b))
201#  define VMADD(a,b,c) ((a)*(b)+(c))
202#  define VSUB(a,b) ((a)-(b))
203#  define LD_PS1(p) (p)
204#  define VALIGNED(ptr) ((((long)(ptr)) & 0x3) == 0)
205#endif
206
207/* shortcuts for complex multiplcations */
208#define VCPLXMUL(ar,ai,br,bi) { v4sf tmp; tmp=VMUL(ar,bi); ar=VMUL(ar,br); ar=VSUB(ar,VMUL(ai,bi)); ai=VMUL(ai,br); ai=VADD(ai,tmp); }
209#define VCPLXMULCONJ(ar,ai,br,bi) { v4sf tmp; tmp=VMUL(ar,bi); ar=VMUL(ar,br); ar=VADD(ar,VMUL(ai,bi)); ai=VMUL(ai,br); ai=VSUB(ai,tmp); }
210#ifndef SVMUL
211/* multiply a scalar with a vector */
212#define SVMUL(f,v) VMUL(LD_PS1(f),v)
213#endif
214
215#if !defined PFFT_MACROS_ONLY
216
217#if !defined(PFFFT_SIMD_DISABLE)
218typedef union v4sf_union {
219  v4sf  v;
220  float f[4];
221} v4sf_union;
222
223#if 0
224#include <string.h>
225
226#define assertv4(v,f0,f1,f2,f3) assert(v.f[0] == (f0) && v.f[1] == (f1) && v.f[2] == (f2) && v.f[3] == (f3))
227
228/* detect bugs with the vector support macros */
229void validate_pffft_simd(void);
230void validate_pffft_simd(void) {
231  float f[16] = { 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 };
232  v4sf_union a0, a1, a2, a3, t, u;
233  memcpy(a0.f, f, 4*sizeof(float));
234  memcpy(a1.f, f+4, 4*sizeof(float));
235  memcpy(a2.f, f+8, 4*sizeof(float));
236  memcpy(a3.f, f+12, 4*sizeof(float));
237
238  t = a0; u = a1; t.v = VZERO();
239  printf("VZERO=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); assertv4(t, 0, 0, 0, 0);
240  t.v = VADD(a1.v, a2.v);
241  printf("VADD(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); assertv4(t, 12, 14, 16, 18);
242  t.v = VMUL(a1.v, a2.v);
243  printf("VMUL(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); assertv4(t, 32, 45, 60, 77);
244  t.v = VMADD(a1.v, a2.v,a0.v);
245  printf("VMADD(4:7,8:11,0:3)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]); assertv4(t, 32, 46, 62, 80);
246  INTERLEAVE2(a1.v,a2.v,t.v,u.v);
247  printf("INTERLEAVE2(4:7,8:11)=[%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3], u.f[0], u.f[1], u.f[2], u.f[3]);
248  assertv4(t, 4, 8, 5, 9); assertv4(u, 6, 10, 7, 11);
249  UNINTERLEAVE2(a1.v,a2.v,t.v,u.v);
250  printf("UNINTERLEAVE2(4:7,8:11)=[%2g %2g %2g %2g] [%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3], u.f[0], u.f[1], u.f[2], u.f[3]);
251  assertv4(t, 4, 6, 8, 10); assertv4(u, 5, 7, 9, 11);
252
253  t.v=LD_PS1(f[15]);
254  printf("LD_PS1(15)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]);
255  assertv4(t, 15, 15, 15, 15);
256  t.v = VSWAPHL(a1.v, a2.v);
257  printf("VSWAPHL(4:7,8:11)=[%2g %2g %2g %2g]\n", t.f[0], t.f[1], t.f[2], t.f[3]);
258  assertv4(t, 8, 9, 6, 7);
259  VTRANSPOSE4(a0.v, a1.v, a2.v, a3.v);
260  printf("VTRANSPOSE4(0:3,4:7,8:11,12:15)=[%2g %2g %2g %2g] [%2g %2g %2g %2g] [%2g %2g %2g %2g] [%2g %2g %2g %2g]\n",
261         a0.f[0], a0.f[1], a0.f[2], a0.f[3], a1.f[0], a1.f[1], a1.f[2], a1.f[3],
262         a2.f[0], a2.f[1], a2.f[2], a2.f[3], a3.f[0], a3.f[1], a3.f[2], a3.f[3]);
263  assertv4(a0, 0, 4, 8, 12); assertv4(a1, 1, 5, 9, 13); assertv4(a2, 2, 6, 10, 14); assertv4(a3, 3, 7, 11, 15);
264}
265#endif
266#endif /*!PFFFT_SIMD_DISABLE */
267
268#if 0
269/* SSE and co like 16-bytes aligned pointers */
270#define MALLOC_V4SF_ALIGNMENT 64 /* with a 64-byte alignment, we are even aligned on L2 cache lines... */
271void *pffft_aligned_malloc(size_t nb_bytes) {
272  void *p, *p0 = malloc(nb_bytes + MALLOC_V4SF_ALIGNMENT);
273  if (!p0) return (void *) 0;
274  p = (void *) (((size_t) p0 + MALLOC_V4SF_ALIGNMENT) & (~((size_t) (MALLOC_V4SF_ALIGNMENT-1))));
275  *((void **) p - 1) = p0;
276  return p;
277}
278
279void pffft_aligned_free(void *p) {
280  if (p) free(*((void **) p - 1));
281}
282
283int pffft_simd_size() { return SIMD_SZ; }
284#endif
285
286/*
287  passf2 and passb2 has been merged here, fsign = -1 for passf2, +1 for passb2
288*/
289static NEVER_INLINE(void) passf2_ps(int ido, int l1, const v4sf *cc, v4sf *ch, const float *wa1, float fsign) {
290  int k, i;
291  int l1ido = l1*ido;
292  if (ido <= 2) {
293    for (k=0; k < l1ido; k += ido, ch += ido, cc+= 2*ido) {
294      ch[0]         = VADD(cc[0], cc[ido+0]);
295      ch[l1ido]     = VSUB(cc[0], cc[ido+0]);
296      ch[1]         = VADD(cc[1], cc[ido+1]);
297      ch[l1ido + 1] = VSUB(cc[1], cc[ido+1]);
298    }
299  } else {
300    for (k=0; k < l1ido; k += ido, ch += ido, cc += 2*ido) {
301      for (i=0; i<ido-1; i+=2) {
302        v4sf tr2 = VSUB(cc[i+0], cc[i+ido+0]);
303        v4sf ti2 = VSUB(cc[i+1], cc[i+ido+1]);
304        v4sf wr = LD_PS1(wa1[i]), wi = VMUL(LD_PS1(fsign), LD_PS1(wa1[i+1]));
305        ch[i]   = VADD(cc[i+0], cc[i+ido+0]);
306        ch[i+1] = VADD(cc[i+1], cc[i+ido+1]);
307        VCPLXMUL(tr2, ti2, wr, wi);
308        ch[i+l1ido]   = tr2;
309        ch[i+l1ido+1] = ti2;
310      }
311    }
312  }
313}
314
315/*
316  passf3 and passb3 has been merged here, fsign = -1 for passf3, +1 for passb3
317*/
318#if 0
319static NEVER_INLINE(void) passf3_ps(int ido, int l1, const v4sf *cc, v4sf *ch,
320                                    const float *wa1, const float *wa2, float fsign) {
321  static const float taur = -0.5f;
322  float taui = 0.866025403784439f*fsign;
323  int i, k;
324  v4sf tr2, ti2, cr2, ci2, cr3, ci3, dr2, di2, dr3, di3;
325  int l1ido = l1*ido;
326  float wr1, wi1, wr2, wi2;
327  assert(ido > 2);
328  for (k=0; k< l1ido; k += ido, cc+= 3*ido, ch +=ido) {
329    for (i=0; i<ido-1; i+=2) {
330      tr2 = VADD(cc[i+ido], cc[i+2*ido]);
331      cr2 = VADD(cc[i], SVMUL(taur,tr2));
332      ch[i]    = VADD(cc[i], tr2);
333      ti2 = VADD(cc[i+ido+1], cc[i+2*ido+1]);
334      ci2 = VADD(cc[i    +1], SVMUL(taur,ti2));
335      ch[i+1]  = VADD(cc[i+1], ti2);
336      cr3 = SVMUL(taui, VSUB(cc[i+ido], cc[i+2*ido]));
337      ci3 = SVMUL(taui, VSUB(cc[i+ido+1], cc[i+2*ido+1]));
338      dr2 = VSUB(cr2, ci3);
339      dr3 = VADD(cr2, ci3);
340      di2 = VADD(ci2, cr3);
341      di3 = VSUB(ci2, cr3);
342      wr1=wa1[i], wi1=fsign*wa1[i+1], wr2=wa2[i], wi2=fsign*wa2[i+1];
343      VCPLXMUL(dr2, di2, LD_PS1(wr1), LD_PS1(wi1));
344      ch[i+l1ido] = dr2;
345      ch[i+l1ido + 1] = di2;
346      VCPLXMUL(dr3, di3, LD_PS1(wr2), LD_PS1(wi2));
347      ch[i+2*l1ido] = dr3;
348      ch[i+2*l1ido+1] = di3;
349    }
350  }
351} /* passf3 */
352#endif
353
354static NEVER_INLINE(void) passf4_ps(int ido, int l1, const v4sf *cc, v4sf *ch,
355                                    const float *wa1, const float *wa2, const float *wa3, float fsign) {
356  /* isign == -1 for forward transform and +1 for backward transform */
357
358  int i, k;
359  v4sf ci2, ci3, ci4, cr2, cr3, cr4, ti1, ti2, ti3, ti4, tr1, tr2, tr3, tr4;
360  int l1ido = l1*ido;
361  if (ido == 2) {
362    for (k=0; k < l1ido; k += ido, ch += ido, cc += 4*ido) {
363      tr1 = VSUB(cc[0], cc[2*ido + 0]);
364      tr2 = VADD(cc[0], cc[2*ido + 0]);
365      ti1 = VSUB(cc[1], cc[2*ido + 1]);
366      ti2 = VADD(cc[1], cc[2*ido + 1]);
367      ti4 = VMUL(VSUB(cc[1*ido + 0], cc[3*ido + 0]), LD_PS1(fsign));
368      tr4 = VMUL(VSUB(cc[3*ido + 1], cc[1*ido + 1]), LD_PS1(fsign));
369      tr3 = VADD(cc[ido + 0], cc[3*ido + 0]);
370      ti3 = VADD(cc[ido + 1], cc[3*ido + 1]);
371
372      ch[0*l1ido + 0] = VADD(tr2, tr3);
373      ch[0*l1ido + 1] = VADD(ti2, ti3);
374      ch[1*l1ido + 0] = VADD(tr1, tr4);
375      ch[1*l1ido + 1] = VADD(ti1, ti4);
376      ch[2*l1ido + 0] = VSUB(tr2, tr3);
377      ch[2*l1ido + 1] = VSUB(ti2, ti3);
378      ch[3*l1ido + 0] = VSUB(tr1, tr4);
379      ch[3*l1ido + 1] = VSUB(ti1, ti4);
380    }
381  } else {
382    for (k=0; k < l1ido; k += ido, ch+=ido, cc += 4*ido) {
383      for (i=0; i<ido-1; i+=2) {
384        float wr1, wi1, wr2, wi2, wr3, wi3;
385        tr1 = VSUB(cc[i + 0], cc[i + 2*ido + 0]);
386        tr2 = VADD(cc[i + 0], cc[i + 2*ido + 0]);
387        ti1 = VSUB(cc[i + 1], cc[i + 2*ido + 1]);
388        ti2 = VADD(cc[i + 1], cc[i + 2*ido + 1]);
389        tr4 = VMUL(VSUB(cc[i + 3*ido + 1], cc[i + 1*ido + 1]), LD_PS1(fsign));
390        ti4 = VMUL(VSUB(cc[i + 1*ido + 0], cc[i + 3*ido + 0]), LD_PS1(fsign));
391        tr3 = VADD(cc[i + ido + 0], cc[i + 3*ido + 0]);
392        ti3 = VADD(cc[i + ido + 1], cc[i + 3*ido + 1]);
393
394        ch[i] = VADD(tr2, tr3);
395        cr3    = VSUB(tr2, tr3);
396        ch[i + 1] = VADD(ti2, ti3);
397        ci3 = VSUB(ti2, ti3);
398
399        cr2 = VADD(tr1, tr4);
400        cr4 = VSUB(tr1, tr4);
401        ci2 = VADD(ti1, ti4);
402        ci4 = VSUB(ti1, ti4);
403        wr1=wa1[i], wi1=fsign*wa1[i+1];
404        VCPLXMUL(cr2, ci2, LD_PS1(wr1), LD_PS1(wi1));
405        wr2=wa2[i], wi2=fsign*wa2[i+1];
406        ch[i + l1ido] = cr2;
407        ch[i + l1ido + 1] = ci2;
408
409        VCPLXMUL(cr3, ci3, LD_PS1(wr2), LD_PS1(wi2));
410        wr3=wa3[i], wi3=fsign*wa3[i+1];
411        ch[i + 2*l1ido] = cr3;
412        ch[i + 2*l1ido + 1] = ci3;
413
414        VCPLXMUL(cr4, ci4, LD_PS1(wr3), LD_PS1(wi3));
415        ch[i + 3*l1ido] = cr4;
416        ch[i + 3*l1ido + 1] = ci4;
417      }
418    }
419  }
420} /* passf4 */
421
422#if 0
423/*
424  passf5 and passb5 has been merged here, fsign = -1 for passf5, +1 for passb5
425*/
426static NEVER_INLINE(void) passf5_ps(int ido, int l1, const v4sf *cc, v4sf *ch,
427                                    const float *wa1, const float *wa2,
428                                    const float *wa3, const float *wa4, float fsign) {
429  static const float tr11 = .309016994374947f;
430  const float ti11 = .951056516295154f*fsign;
431  static const float tr12 = -.809016994374947f;
432  const float ti12 = .587785252292473f*fsign;
433
434  /* Local variables */
435  int i, k;
436  v4sf ci2, ci3, ci4, ci5, di3, di4, di5, di2, cr2, cr3, cr5, cr4, ti2, ti3,
437    ti4, ti5, dr3, dr4, dr5, dr2, tr2, tr3, tr4, tr5;
438
439  float wr1, wi1, wr2, wi2, wr3, wi3, wr4, wi4;
440
441#define cc_ref(a_1,a_2) cc[(a_2-1)*ido + a_1 + 1]
442#define ch_ref(a_1,a_3) ch[(a_3-1)*l1*ido + a_1 + 1]
443
444  assert(ido > 2);
445  for (k = 0; k < l1; ++k, cc += 5*ido, ch += ido) {
446    for (i = 0; i < ido-1; i += 2) {
447      ti5 = VSUB(cc_ref(i  , 2), cc_ref(i  , 5));
448      ti2 = VADD(cc_ref(i  , 2), cc_ref(i  , 5));
449      ti4 = VSUB(cc_ref(i  , 3), cc_ref(i  , 4));
450      ti3 = VADD(cc_ref(i  , 3), cc_ref(i  , 4));
451      tr5 = VSUB(cc_ref(i-1, 2), cc_ref(i-1, 5));
452      tr2 = VADD(cc_ref(i-1, 2), cc_ref(i-1, 5));
453      tr4 = VSUB(cc_ref(i-1, 3), cc_ref(i-1, 4));
454      tr3 = VADD(cc_ref(i-1, 3), cc_ref(i-1, 4));
455      ch_ref(i-1, 1) = VADD(cc_ref(i-1, 1), VADD(tr2, tr3));
456      ch_ref(i  , 1) = VADD(cc_ref(i  , 1), VADD(ti2, ti3));
457      cr2 = VADD(cc_ref(i-1, 1), VADD(SVMUL(tr11, tr2),SVMUL(tr12, tr3)));
458      ci2 = VADD(cc_ref(i  , 1), VADD(SVMUL(tr11, ti2),SVMUL(tr12, ti3)));
459      cr3 = VADD(cc_ref(i-1, 1), VADD(SVMUL(tr12, tr2),SVMUL(tr11, tr3)));
460      ci3 = VADD(cc_ref(i  , 1), VADD(SVMUL(tr12, ti2),SVMUL(tr11, ti3)));
461      cr5 = VADD(SVMUL(ti11, tr5), SVMUL(ti12, tr4));
462      ci5 = VADD(SVMUL(ti11, ti5), SVMUL(ti12, ti4));
463      cr4 = VSUB(SVMUL(ti12, tr5), SVMUL(ti11, tr4));
464      ci4 = VSUB(SVMUL(ti12, ti5), SVMUL(ti11, ti4));
465      dr3 = VSUB(cr3, ci4);
466      dr4 = VADD(cr3, ci4);
467      di3 = VADD(ci3, cr4);
468      di4 = VSUB(ci3, cr4);
469      dr5 = VADD(cr2, ci5);
470      dr2 = VSUB(cr2, ci5);
471      di5 = VSUB(ci2, cr5);
472      di2 = VADD(ci2, cr5);
473      wr1=wa1[i], wi1=fsign*wa1[i+1], wr2=wa2[i], wi2=fsign*wa2[i+1];
474      wr3=wa3[i], wi3=fsign*wa3[i+1], wr4=wa4[i], wi4=fsign*wa4[i+1];
475      VCPLXMUL(dr2, di2, LD_PS1(wr1), LD_PS1(wi1));
476      ch_ref(i - 1, 2) = dr2;
477      ch_ref(i, 2)     = di2;
478      VCPLXMUL(dr3, di3, LD_PS1(wr2), LD_PS1(wi2));
479      ch_ref(i - 1, 3) = dr3;
480      ch_ref(i, 3)     = di3;
481      VCPLXMUL(dr4, di4, LD_PS1(wr3), LD_PS1(wi3));
482      ch_ref(i - 1, 4) = dr4;
483      ch_ref(i, 4)     = di4;
484      VCPLXMUL(dr5, di5, LD_PS1(wr4), LD_PS1(wi4));
485      ch_ref(i - 1, 5) = dr5;
486      ch_ref(i, 5)     = di5;
487    }
488  }
489#undef ch_ref
490#undef cc_ref
491}
492#endif
493
494static NEVER_INLINE(void) radf2_ps(int ido, int l1, const v4sf * RESTRICT cc, v4sf * RESTRICT ch, const float *wa1) {
495  static const float minus_one = -1.f;
496  int i, k, l1ido = l1*ido;
497  for (k=0; k < l1ido; k += ido) {
498    v4sf a = cc[k], b = cc[k + l1ido];
499    ch[2*k] = VADD(a, b);
500    ch[2*(k+ido)-1] = VSUB(a, b);
501  }
502  if (ido < 2) return;
503  if (ido != 2) {
504    for (k=0; k < l1ido; k += ido) {
505      for (i=2; i<ido; i+=2) {
506        v4sf tr2 = cc[i - 1 + k + l1ido], ti2 = cc[i + k + l1ido];
507        v4sf br = cc[i - 1 + k], bi = cc[i + k];
508        VCPLXMULCONJ(tr2, ti2, LD_PS1(wa1[i - 2]), LD_PS1(wa1[i - 1]));
509        ch[i + 2*k] = VADD(bi, ti2);
510        ch[2*(k+ido) - i] = VSUB(ti2, bi);
511        ch[i - 1 + 2*k] = VADD(br, tr2);
512        ch[2*(k+ido) - i -1] = VSUB(br, tr2);
513      }
514    }
515    if (ido % 2 == 1) return;
516  }
517  for (k=0; k < l1ido; k += ido) {
518    ch[2*k + ido] = SVMUL(minus_one, cc[ido-1 + k + l1ido]);
519    ch[2*k + ido-1] = cc[k + ido-1];
520  }
521} /* radf2 */
522
523
524static NEVER_INLINE(void) radb2_ps(int ido, int l1, const v4sf *cc, v4sf *ch, const float *wa1) {
525  static const float minus_two=-2;
526  int i, k, l1ido = l1*ido;
527  v4sf a,b,c,d, tr2, ti2;
528  for (k=0; k < l1ido; k += ido) {
529    a = cc[2*k]; b = cc[2*(k+ido) - 1];
530    ch[k] = VADD(a, b);
531    ch[k + l1ido] =VSUB(a, b);
532  }
533  if (ido < 2) return;
534  if (ido != 2) {
535    for (k = 0; k < l1ido; k += ido) {
536      for (i = 2; i < ido; i += 2) {
537        a = cc[i-1 + 2*k]; b = cc[2*(k + ido) - i - 1];
538        c = cc[i+0 + 2*k]; d = cc[2*(k + ido) - i + 0];
539        ch[i-1 + k] = VADD(a, b);
540        tr2 = VSUB(a, b);
541        ch[i+0 + k] = VSUB(c, d);
542        ti2 = VADD(c, d);
543        VCPLXMUL(tr2, ti2, LD_PS1(wa1[i - 2]), LD_PS1(wa1[i - 1]));
544        ch[i-1 + k + l1ido] = tr2;
545        ch[i+0 + k + l1ido] = ti2;
546      }
547    }
548    if (ido % 2 == 1) return;
549  }
550  for (k = 0; k < l1ido; k += ido) {
551    a = cc[2*k + ido-1]; b = cc[2*k + ido];
552    ch[k + ido-1] = VADD(a,a);
553    ch[k + ido-1 + l1ido] = SVMUL(minus_two, b);
554  }
555} /* radb2 */
556
557#if 0
558static void radf3_ps(int ido, int l1, const v4sf * RESTRICT cc, v4sf * RESTRICT ch,
559                     const float *wa1, const float *wa2) {
560  static const float taur = -0.5f;
561  static const float taui = 0.866025403784439f;
562  int i, k, ic;
563  v4sf ci2, di2, di3, cr2, dr2, dr3, ti2, ti3, tr2, tr3, wr1, wi1, wr2, wi2;
564  for (k=0; k<l1; k++) {
565    cr2 = VADD(cc[(k + l1)*ido], cc[(k + 2*l1)*ido]);
566    ch[3*k*ido] = VADD(cc[k*ido], cr2);
567    ch[(3*k+2)*ido] = SVMUL(taui, VSUB(cc[(k + l1*2)*ido], cc[(k + l1)*ido]));
568    ch[ido-1 + (3*k + 1)*ido] = VADD(cc[k*ido], SVMUL(taur, cr2));
569  }
570  if (ido == 1) return;
571  for (k=0; k<l1; k++) {
572    for (i=2; i<ido; i+=2) {
573      ic = ido - i;
574      wr1 = LD_PS1(wa1[i - 2]); wi1 = LD_PS1(wa1[i - 1]);
575      dr2 = cc[i - 1 + (k + l1)*ido]; di2 = cc[i + (k + l1)*ido];
576      VCPLXMULCONJ(dr2, di2, wr1, wi1);
577
578      wr2 = LD_PS1(wa2[i - 2]); wi2 = LD_PS1(wa2[i - 1]);
579      dr3 = cc[i - 1 + (k + l1*2)*ido]; di3 = cc[i + (k + l1*2)*ido];
580      VCPLXMULCONJ(dr3, di3, wr2, wi2);
581
582      cr2 = VADD(dr2, dr3);
583      ci2 = VADD(di2, di3);
584      ch[i - 1 + 3*k*ido] = VADD(cc[i - 1 + k*ido], cr2);
585      ch[i + 3*k*ido] = VADD(cc[i + k*ido], ci2);
586      tr2 = VADD(cc[i - 1 + k*ido], SVMUL(taur, cr2));
587      ti2 = VADD(cc[i + k*ido], SVMUL(taur, ci2));
588      tr3 = SVMUL(taui, VSUB(di2, di3));
589      ti3 = SVMUL(taui, VSUB(dr3, dr2));
590      ch[i - 1 + (3*k + 2)*ido] = VADD(tr2, tr3);
591      ch[ic - 1 + (3*k + 1)*ido] = VSUB(tr2, tr3);
592      ch[i + (3*k + 2)*ido] = VADD(ti2, ti3);
593      ch[ic + (3*k + 1)*ido] = VSUB(ti3, ti2);
594    }
595  }
596} /* radf3 */
597
598
599static void radb3_ps(int ido, int l1, const v4sf *RESTRICT cc, v4sf *RESTRICT ch,
600                     const float *wa1, const float *wa2)
601{
602  static const float taur = -0.5f;
603  static const float taui = 0.866025403784439f;
604  static const float taui_2 = 0.866025403784439f*2;
605  int i, k, ic;
606  v4sf ci2, ci3, di2, di3, cr2, cr3, dr2, dr3, ti2, tr2;
607  for (k=0; k<l1; k++) {
608    tr2 = cc[ido-1 + (3*k + 1)*ido]; tr2 = VADD(tr2,tr2);
609    cr2 = VMADD(LD_PS1(taur), tr2, cc[3*k*ido]);
610    ch[k*ido] = VADD(cc[3*k*ido], tr2);
611    ci3 = SVMUL(taui_2, cc[(3*k + 2)*ido]);
612    ch[(k + l1)*ido] = VSUB(cr2, ci3);
613    ch[(k + 2*l1)*ido] = VADD(cr2, ci3);
614  }
615  if (ido == 1) return;
616  for (k=0; k<l1; k++) {
617    for (i=2; i<ido; i+=2) {
618      ic = ido - i;
619      tr2 = VADD(cc[i - 1 + (3*k + 2)*ido], cc[ic - 1 + (3*k + 1)*ido]);
620      cr2 = VMADD(LD_PS1(taur), tr2, cc[i - 1 + 3*k*ido]);
621      ch[i - 1 + k*ido] = VADD(cc[i - 1 + 3*k*ido], tr2);
622      ti2 = VSUB(cc[i + (3*k + 2)*ido], cc[ic + (3*k + 1)*ido]);
623      ci2 = VMADD(LD_PS1(taur), ti2, cc[i + 3*k*ido]);
624      ch[i + k*ido] = VADD(cc[i + 3*k*ido], ti2);
625      cr3 = SVMUL(taui, VSUB(cc[i - 1 + (3*k + 2)*ido], cc[ic - 1 + (3*k + 1)*ido]));
626      ci3 = SVMUL(taui, VADD(cc[i + (3*k + 2)*ido], cc[ic + (3*k + 1)*ido]));
627      dr2 = VSUB(cr2, ci3);
628      dr3 = VADD(cr2, ci3);
629      di2 = VADD(ci2, cr3);
630      di3 = VSUB(ci2, cr3);
631      VCPLXMUL(dr2, di2, LD_PS1(wa1[i-2]), LD_PS1(wa1[i-1]));
632      ch[i - 1 + (k + l1)*ido] = dr2;
633      ch[i + (k + l1)*ido] = di2;
634      VCPLXMUL(dr3, di3, LD_PS1(wa2[i-2]), LD_PS1(wa2[i-1]));
635      ch[i - 1 + (k + 2*l1)*ido] = dr3;
636      ch[i + (k + 2*l1)*ido] = di3;
637    }
638  }
639} /* radb3 */
640#endif
641
642static NEVER_INLINE(void) radf4_ps(int ido, int l1, const v4sf *RESTRICT cc, v4sf * RESTRICT ch,
643                                   const float * RESTRICT wa1, const float * RESTRICT wa2, const float * RESTRICT wa3)
644{
645  static const float minus_hsqt2 = (float)-0.7071067811865475;
646  int i, k, l1ido = l1*ido;
647  {
648    const v4sf *RESTRICT cc_ = cc, * RESTRICT cc_end = cc + l1ido;
649    v4sf * RESTRICT ch_ = ch;
650    while (cc < cc_end) {
651      /* this loop represents between 25% and 40% of total radf4_ps cost ! */
652      v4sf a0 = cc[0], a1 = cc[l1ido];
653      v4sf a2 = cc[2*l1ido], a3 = cc[3*l1ido];
654      v4sf tr1 = VADD(a1, a3);
655      v4sf tr2 = VADD(a0, a2);
656      ch[2*ido-1] = VSUB(a0, a2);
657      ch[2*ido  ] = VSUB(a3, a1);
658      ch[0      ] = VADD(tr1, tr2);
659      ch[4*ido-1] = VSUB(tr2, tr1);
660      cc += ido; ch += 4*ido;
661    }
662    cc = cc_; ch = ch_;
663  }
664  if (ido < 2) return;
665  if (ido != 2) {
666    for (k = 0; k < l1ido; k += ido) {
667      const v4sf * RESTRICT pc = (v4sf*)(cc + 1 + k);
668      for (i=2; i<ido; i += 2, pc += 2) {
669        int ic = ido - i;
670        v4sf wr, wi, cr2, ci2, cr3, ci3, cr4, ci4;
671        v4sf tr1, ti1, tr2, ti2, tr3, ti3, tr4, ti4;
672
673        cr2 = pc[1*l1ido+0];
674        ci2 = pc[1*l1ido+1];
675        wr=LD_PS1(wa1[i - 2]);
676        wi=LD_PS1(wa1[i - 1]);
677        VCPLXMULCONJ(cr2,ci2,wr,wi);
678
679        cr3 = pc[2*l1ido+0];
680        ci3 = pc[2*l1ido+1];
681        wr = LD_PS1(wa2[i-2]);
682        wi = LD_PS1(wa2[i-1]);
683        VCPLXMULCONJ(cr3, ci3, wr, wi);
684
685        cr4 = pc[3*l1ido];
686        ci4 = pc[3*l1ido+1];
687        wr = LD_PS1(wa3[i-2]);
688        wi = LD_PS1(wa3[i-1]);
689        VCPLXMULCONJ(cr4, ci4, wr, wi);
690
691        /* at this point, on SSE, five of "cr2 cr3 cr4 ci2 ci3 ci4" should be loaded in registers */
692
693        tr1 = VADD(cr2,cr4);
694        tr4 = VSUB(cr4,cr2);
695        tr2 = VADD(pc[0],cr3);
696        tr3 = VSUB(pc[0],cr3);
697        ch[i - 1 + 4*k] = VADD(tr1,tr2);
698        ch[ic - 1 + 4*k + 3*ido] = VSUB(tr2,tr1); /* at this point tr1 and tr2 can be disposed */
699        ti1 = VADD(ci2,ci4);
700        ti4 = VSUB(ci2,ci4);
701        ch[i - 1 + 4*k + 2*ido] = VADD(ti4,tr3);
702        ch[ic - 1 + 4*k + 1*ido] = VSUB(tr3,ti4); /* dispose tr3, ti4 */
703        ti2 = VADD(pc[1],ci3);
704        ti3 = VSUB(pc[1],ci3);
705        ch[i + 4*k] = VADD(ti1, ti2);
706        ch[ic + 4*k + 3*ido] = VSUB(ti1, ti2);
707        ch[i + 4*k + 2*ido] = VADD(tr4, ti3);
708        ch[ic + 4*k + 1*ido] = VSUB(tr4, ti3);
709      }
710    }
711    if (ido % 2 == 1) return;
712  }
713  for (k=0; k<l1ido; k += ido) {
714    v4sf a = cc[ido-1 + k + l1ido], b = cc[ido-1 + k + 3*l1ido];
715    v4sf c = cc[ido-1 + k], d = cc[ido-1 + k + 2*l1ido];
716    v4sf ti1 = SVMUL(minus_hsqt2, VADD(a, b));
717    v4sf tr1 = SVMUL(minus_hsqt2, VSUB(b, a));
718    ch[ido-1 + 4*k] = VADD(tr1, c);
719    ch[ido-1 + 4*k + 2*ido] = VSUB(c, tr1);
720    ch[4*k + 1*ido] = VSUB(ti1, d);
721    ch[4*k + 3*ido] = VADD(ti1, d);
722  }
723} /* radf4 */
724
725
726static NEVER_INLINE(void) radb4_ps(int ido, int l1, const v4sf * RESTRICT cc, v4sf * RESTRICT ch,
727                                   const float * RESTRICT wa1, const float * RESTRICT wa2, const float *RESTRICT wa3)
728{
729  static const float minus_sqrt2 = (float)-1.414213562373095;
730  static const float two = 2.f;
731  int i, k, l1ido = l1*ido;
732  v4sf ci2, ci3, ci4, cr2, cr3, cr4, ti1, ti2, ti3, ti4, tr1, tr2, tr3, tr4;
733  {
734    const v4sf *RESTRICT cc_ = cc, * RESTRICT ch_end = ch + l1ido;
735    v4sf *ch_ = ch;
736    while (ch < ch_end) {
737      v4sf a = cc[0], b = cc[4*ido-1];
738      v4sf c = cc[2*ido], d = cc[2*ido-1];
739      tr3 = SVMUL(two,d);
740      tr2 = VADD(a,b);
741      tr1 = VSUB(a,b);
742      tr4 = SVMUL(two,c);
743      ch[0*l1ido] = VADD(tr2, tr3);
744      ch[2*l1ido] = VSUB(tr2, tr3);
745      ch[1*l1ido] = VSUB(tr1, tr4);
746      ch[3*l1ido] = VADD(tr1, tr4);
747
748      cc += 4*ido; ch += ido;
749    }
750    cc = cc_; ch = ch_;
751  }
752  if (ido < 2) return;
753  if (ido != 2) {
754    for (k = 0; k < l1ido; k += ido) {
755      const v4sf * RESTRICT pc = (v4sf*)(cc - 1 + 4*k);
756      v4sf * RESTRICT ph = (v4sf*)(ch + k + 1);
757      for (i = 2; i < ido; i += 2) {
758
759        tr1 = VSUB(pc[i], pc[4*ido - i]);
760        tr2 = VADD(pc[i], pc[4*ido - i]);
761        ti4 = VSUB(pc[2*ido + i], pc[2*ido - i]);
762        tr3 = VADD(pc[2*ido + i], pc[2*ido - i]);
763        ph[0] = VADD(tr2, tr3);
764        cr3 = VSUB(tr2, tr3);
765
766        ti3 = VSUB(pc[2*ido + i + 1], pc[2*ido - i + 1]);
767        tr4 = VADD(pc[2*ido + i + 1], pc[2*ido - i + 1]);
768        cr2 = VSUB(tr1, tr4);
769        cr4 = VADD(tr1, tr4);
770
771        ti1 = VADD(pc[i + 1], pc[4*ido - i + 1]);
772        ti2 = VSUB(pc[i + 1], pc[4*ido - i + 1]);
773
774        ph[1] = VADD(ti2, ti3); ph += l1ido;
775        ci3 = VSUB(ti2, ti3);
776        ci2 = VADD(ti1, ti4);
777        ci4 = VSUB(ti1, ti4);
778        VCPLXMUL(cr2, ci2, LD_PS1(wa1[i-2]), LD_PS1(wa1[i-1]));
779        ph[0] = cr2;
780        ph[1] = ci2; ph += l1ido;
781        VCPLXMUL(cr3, ci3, LD_PS1(wa2[i-2]), LD_PS1(wa2[i-1]));
782        ph[0] = cr3;
783        ph[1] = ci3; ph += l1ido;
784        VCPLXMUL(cr4, ci4, LD_PS1(wa3[i-2]), LD_PS1(wa3[i-1]));
785        ph[0] = cr4;
786        ph[1] = ci4; ph = ph - 3*l1ido + 2;
787      }
788    }
789    if (ido % 2 == 1) return;
790  }
791  for (k=0; k < l1ido; k+=ido) {
792    int i0 = 4*k + ido;
793    v4sf c = cc[i0-1], d = cc[i0 + 2*ido-1];
794    v4sf a = cc[i0+0], b = cc[i0 + 2*ido+0];
795    tr1 = VSUB(c,d);
796    tr2 = VADD(c,d);
797    ti1 = VADD(b,a);
798    ti2 = VSUB(b,a);
799    ch[ido-1 + k + 0*l1ido] = VADD(tr2,tr2);
800    ch[ido-1 + k + 1*l1ido] = SVMUL(minus_sqrt2, VSUB(ti1, tr1));
801    ch[ido-1 + k + 2*l1ido] = VADD(ti2, ti2);
802    ch[ido-1 + k + 3*l1ido] = SVMUL(minus_sqrt2, VADD(ti1, tr1));
803  }
804} /* radb4 */
805
806#if 0
807static void radf5_ps(int ido, int l1, const v4sf * RESTRICT cc, v4sf * RESTRICT ch,
808                     const float *wa1, const float *wa2, const float *wa3, const float *wa4)
809{
810  static const float tr11 = .309016994374947f;
811  static const float ti11 = .951056516295154f;
812  static const float tr12 = -.809016994374947f;
813  static const float ti12 = .587785252292473f;
814
815  /* System generated locals */
816  int cc_offset, ch_offset;
817
818  /* Local variables */
819  int i, k, ic;
820  v4sf ci2, di2, ci4, ci5, di3, di4, di5, ci3, cr2, cr3, dr2, dr3, dr4, dr5,
821    cr5, cr4, ti2, ti3, ti5, ti4, tr2, tr3, tr4, tr5;
822  int idp2;
823
824
825#define cc_ref(a_1,a_2,a_3) cc[((a_3)*l1 + (a_2))*ido + a_1]
826#define ch_ref(a_1,a_2,a_3) ch[((a_3)*5 + (a_2))*ido + a_1]
827
828  /* Parameter adjustments */
829  ch_offset = 1 + ido * 6;
830  ch -= ch_offset;
831  cc_offset = 1 + ido * (1 + l1);
832  cc -= cc_offset;
833
834  /* Function Body */
835  for (k = 1; k <= l1; ++k) {
836    cr2 = VADD(cc_ref(1, k, 5), cc_ref(1, k, 2));
837    ci5 = VSUB(cc_ref(1, k, 5), cc_ref(1, k, 2));
838    cr3 = VADD(cc_ref(1, k, 4), cc_ref(1, k, 3));
839    ci4 = VSUB(cc_ref(1, k, 4), cc_ref(1, k, 3));
840    ch_ref(1, 1, k) = VADD(cc_ref(1, k, 1), VADD(cr2, cr3));
841    ch_ref(ido, 2, k) = VADD(cc_ref(1, k, 1), VADD(SVMUL(tr11, cr2), SVMUL(tr12, cr3)));
842    ch_ref(1, 3, k) = VADD(SVMUL(ti11, ci5), SVMUL(ti12, ci4));
843    ch_ref(ido, 4, k) = VADD(cc_ref(1, k, 1), VADD(SVMUL(tr12, cr2), SVMUL(tr11, cr3)));
844    ch_ref(1, 5, k) = VSUB(SVMUL(ti12, ci5), SVMUL(ti11, ci4));
845    /*printf("pffft: radf5, k=%d ch_ref=%f, ci4=%f\n", k, ch_ref(1, 5, k), ci4); */
846  }
847  if (ido == 1) {
848    return;
849  }
850  idp2 = ido + 2;
851  for (k = 1; k <= l1; ++k) {
852    for (i = 3; i <= ido; i += 2) {
853      ic = idp2 - i;
854      dr2 = LD_PS1(wa1[i-3]); di2 = LD_PS1(wa1[i-2]);
855      dr3 = LD_PS1(wa2[i-3]); di3 = LD_PS1(wa2[i-2]);
856      dr4 = LD_PS1(wa3[i-3]); di4 = LD_PS1(wa3[i-2]);
857      dr5 = LD_PS1(wa4[i-3]); di5 = LD_PS1(wa4[i-2]);
858      VCPLXMULCONJ(dr2, di2, cc_ref(i-1, k, 2), cc_ref(i, k, 2));
859      VCPLXMULCONJ(dr3, di3, cc_ref(i-1, k, 3), cc_ref(i, k, 3));
860      VCPLXMULCONJ(dr4, di4, cc_ref(i-1, k, 4), cc_ref(i, k, 4));
861      VCPLXMULCONJ(dr5, di5, cc_ref(i-1, k, 5), cc_ref(i, k, 5));
862      cr2 = VADD(dr2, dr5);
863      ci5 = VSUB(dr5, dr2);
864      cr5 = VSUB(di2, di5);
865      ci2 = VADD(di2, di5);
866      cr3 = VADD(dr3, dr4);
867      ci4 = VSUB(dr4, dr3);
868      cr4 = VSUB(di3, di4);
869      ci3 = VADD(di3, di4);
870      ch_ref(i - 1, 1, k) = VADD(cc_ref(i - 1, k, 1), VADD(cr2, cr3));
871      ch_ref(i, 1, k) = VSUB(cc_ref(i, k, 1), VADD(ci2, ci3));/* */
872      tr2 = VADD(cc_ref(i - 1, k, 1), VADD(SVMUL(tr11, cr2), SVMUL(tr12, cr3)));
873      ti2 = VSUB(cc_ref(i, k, 1), VADD(SVMUL(tr11, ci2), SVMUL(tr12, ci3)));/* */
874      tr3 = VADD(cc_ref(i - 1, k, 1), VADD(SVMUL(tr12, cr2), SVMUL(tr11, cr3)));
875      ti3 = VSUB(cc_ref(i, k, 1), VADD(SVMUL(tr12, ci2), SVMUL(tr11, ci3)));/* */
876      tr5 = VADD(SVMUL(ti11, cr5), SVMUL(ti12, cr4));
877      ti5 = VADD(SVMUL(ti11, ci5), SVMUL(ti12, ci4));
878      tr4 = VSUB(SVMUL(ti12, cr5), SVMUL(ti11, cr4));
879      ti4 = VSUB(SVMUL(ti12, ci5), SVMUL(ti11, ci4));
880      ch_ref(i - 1, 3, k) = VSUB(tr2, tr5);
881      ch_ref(ic - 1, 2, k) = VADD(tr2, tr5);
882      ch_ref(i, 3, k) = VADD(ti2, ti5);
883      ch_ref(ic, 2, k) = VSUB(ti5, ti2);
884      ch_ref(i - 1, 5, k) = VSUB(tr3, tr4);
885      ch_ref(ic - 1, 4, k) = VADD(tr3, tr4);
886      ch_ref(i, 5, k) = VADD(ti3, ti4);
887      ch_ref(ic, 4, k) = VSUB(ti4, ti3);
888    }
889  }
890#undef cc_ref
891#undef ch_ref
892} /* radf5 */
893
894static void radb5_ps(int ido, int l1, const v4sf *RESTRICT cc, v4sf *RESTRICT ch,
895                  const float *wa1, const float *wa2, const float *wa3, const float *wa4)
896{
897  static const float tr11 = .309016994374947f;
898  static const float ti11 = .951056516295154f;
899  static const float tr12 = -.809016994374947f;
900  static const float ti12 = .587785252292473f;
901
902  int cc_offset, ch_offset;
903
904  /* Local variables */
905  int i, k, ic;
906  v4sf ci2, ci3, ci4, ci5, di3, di4, di5, di2, cr2, cr3, cr5, cr4, ti2, ti3,
907    ti4, ti5, dr3, dr4, dr5, dr2, tr2, tr3, tr4, tr5;
908  int idp2;
909
910#define cc_ref(a_1,a_2,a_3) cc[((a_3)*5 + (a_2))*ido + a_1]
911#define ch_ref(a_1,a_2,a_3) ch[((a_3)*l1 + (a_2))*ido + a_1]
912
913  /* Parameter adjustments */
914  ch_offset = 1 + ido * (1 + l1);
915  ch -= ch_offset;
916  cc_offset = 1 + ido * 6;
917  cc -= cc_offset;
918
919  /* Function Body */
920  for (k = 1; k <= l1; ++k) {
921    ti5 = VADD(cc_ref(1, 3, k), cc_ref(1, 3, k));
922    ti4 = VADD(cc_ref(1, 5, k), cc_ref(1, 5, k));
923    tr2 = VADD(cc_ref(ido, 2, k), cc_ref(ido, 2, k));
924    tr3 = VADD(cc_ref(ido, 4, k), cc_ref(ido, 4, k));
925    ch_ref(1, k, 1) = VADD(cc_ref(1, 1, k), VADD(tr2, tr3));
926    cr2 = VADD(cc_ref(1, 1, k), VADD(SVMUL(tr11, tr2), SVMUL(tr12, tr3)));
927    cr3 = VADD(cc_ref(1, 1, k), VADD(SVMUL(tr12, tr2), SVMUL(tr11, tr3)));
928    ci5 = VADD(SVMUL(ti11, ti5), SVMUL(ti12, ti4));
929    ci4 = VSUB(SVMUL(ti12, ti5), SVMUL(ti11, ti4));
930    ch_ref(1, k, 2) = VSUB(cr2, ci5);
931    ch_ref(1, k, 3) = VSUB(cr3, ci4);
932    ch_ref(1, k, 4) = VADD(cr3, ci4);
933    ch_ref(1, k, 5) = VADD(cr2, ci5);
934  }
935  if (ido == 1) {
936    return;
937  }
938  idp2 = ido + 2;
939  for (k = 1; k <= l1; ++k) {
940    for (i = 3; i <= ido; i += 2) {
941      ic = idp2 - i;
942      ti5 = VADD(cc_ref(i  , 3, k), cc_ref(ic  , 2, k));
943      ti2 = VSUB(cc_ref(i  , 3, k), cc_ref(ic  , 2, k));
944      ti4 = VADD(cc_ref(i  , 5, k), cc_ref(ic  , 4, k));
945      ti3 = VSUB(cc_ref(i  , 5, k), cc_ref(ic  , 4, k));
946      tr5 = VSUB(cc_ref(i-1, 3, k), cc_ref(ic-1, 2, k));
947      tr2 = VADD(cc_ref(i-1, 3, k), cc_ref(ic-1, 2, k));
948      tr4 = VSUB(cc_ref(i-1, 5, k), cc_ref(ic-1, 4, k));
949      tr3 = VADD(cc_ref(i-1, 5, k), cc_ref(ic-1, 4, k));
950      ch_ref(i - 1, k, 1) = VADD(cc_ref(i-1, 1, k), VADD(tr2, tr3));
951      ch_ref(i, k, 1) = VADD(cc_ref(i, 1, k), VADD(ti2, ti3));
952      cr2 = VADD(cc_ref(i-1, 1, k), VADD(SVMUL(tr11, tr2), SVMUL(tr12, tr3)));
953      ci2 = VADD(cc_ref(i  , 1, k), VADD(SVMUL(tr11, ti2), SVMUL(tr12, ti3)));
954      cr3 = VADD(cc_ref(i-1, 1, k), VADD(SVMUL(tr12, tr2), SVMUL(tr11, tr3)));
955      ci3 = VADD(cc_ref(i  , 1, k), VADD(SVMUL(tr12, ti2), SVMUL(tr11, ti3)));
956      cr5 = VADD(SVMUL(ti11, tr5), SVMUL(ti12, tr4));
957      ci5 = VADD(SVMUL(ti11, ti5), SVMUL(ti12, ti4));
958      cr4 = VSUB(SVMUL(ti12, tr5), SVMUL(ti11, tr4));
959      ci4 = VSUB(SVMUL(ti12, ti5), SVMUL(ti11, ti4));
960      dr3 = VSUB(cr3, ci4);
961      dr4 = VADD(cr3, ci4);
962      di3 = VADD(ci3, cr4);
963      di4 = VSUB(ci3, cr4);
964      dr5 = VADD(cr2, ci5);
965      dr2 = VSUB(cr2, ci5);
966      di5 = VSUB(ci2, cr5);
967      di2 = VADD(ci2, cr5);
968      VCPLXMUL(dr2, di2, LD_PS1(wa1[i-3]), LD_PS1(wa1[i-2]));
969      VCPLXMUL(dr3, di3, LD_PS1(wa2[i-3]), LD_PS1(wa2[i-2]));
970      VCPLXMUL(dr4, di4, LD_PS1(wa3[i-3]), LD_PS1(wa3[i-2]));
971      VCPLXMUL(dr5, di5, LD_PS1(wa4[i-3]), LD_PS1(wa4[i-2]));
972
973      ch_ref(i-1, k, 2) = dr2; ch_ref(i, k, 2) = di2;
974      ch_ref(i-1, k, 3) = dr3; ch_ref(i, k, 3) = di3;
975      ch_ref(i-1, k, 4) = dr4; ch_ref(i, k, 4) = di4;
976      ch_ref(i-1, k, 5) = dr5; ch_ref(i, k, 5) = di5;
977    }
978  }
979#undef cc_ref
980#undef ch_ref
981} /* radb5 */
982#endif
983
984static NEVER_INLINE(v4sf *) rfftf1_ps(int n, const v4sf *input_readonly, v4sf *work1, v4sf *work2,
985                                      const float *wa, const int *ifac) {
986  v4sf *in  = (v4sf*)input_readonly;
987  v4sf *out = (in == work2 ? work1 : work2);
988  int nf = ifac[1], k1;
989  int l2 = n;
990  int iw = n-1;
991  assert(in != out && work1 != work2);
992  for (k1 = 1; k1 <= nf; ++k1) {
993    int kh = nf - k1;
994    int ip = ifac[kh + 2];
995    int l1 = l2 / ip;
996    int ido = n / l2;
997    iw -= (ip - 1)*ido;
998    switch (ip) {
999#if 0
1000      case 5: {
1001        int ix2 = iw + ido;
1002        int ix3 = ix2 + ido;
1003        int ix4 = ix3 + ido;
1004        radf5_ps(ido, l1, in, out, &wa[iw], &wa[ix2], &wa[ix3], &wa[ix4]);
1005      } break;
1006#endif
1007      case 4: {
1008        int ix2 = iw + ido;
1009        int ix3 = ix2 + ido;
1010        radf4_ps(ido, l1, in, out, &wa[iw], &wa[ix2], &wa[ix3]);
1011      } break;
1012#if 0
1013      case 3: {
1014        int ix2 = iw + ido;
1015        radf3_ps(ido, l1, in, out, &wa[iw], &wa[ix2]);
1016      } break;
1017#endif
1018      case 2:
1019        radf2_ps(ido, l1, in, out, &wa[iw]);
1020        break;
1021      default:
1022        assert(0);
1023        break;
1024    }
1025    l2 = l1;
1026    if (out == work2) {
1027      out = work1; in = work2;
1028    } else {
1029      out = work2; in = work1;
1030    }
1031  }
1032  return in; /* this is in fact the output .. */
1033} /* rfftf1 */
1034
1035static NEVER_INLINE(v4sf *) rfftb1_ps(int n, const v4sf *input_readonly, v4sf *work1, v4sf *work2,
1036                                      const float *wa, const int *ifac) {
1037  v4sf *in  = (v4sf*)input_readonly;
1038  v4sf *out = (in == work2 ? work1 : work2);
1039  int nf = ifac[1], k1;
1040  int l1 = 1;
1041  int iw = 0;
1042  assert(in != out);
1043  for (k1=1; k1<=nf; k1++) {
1044    int ip = ifac[k1 + 1];
1045    int l2 = ip*l1;
1046    int ido = n / l2;
1047    switch (ip) {
1048#if 0
1049      case 5: {
1050        int ix2 = iw + ido;
1051        int ix3 = ix2 + ido;
1052        int ix4 = ix3 + ido;
1053        radb5_ps(ido, l1, in, out, &wa[iw], &wa[ix2], &wa[ix3], &wa[ix4]);
1054      } break;
1055#endif
1056      case 4: {
1057        int ix2 = iw + ido;
1058        int ix3 = ix2 + ido;
1059        radb4_ps(ido, l1, in, out, &wa[iw], &wa[ix2], &wa[ix3]);
1060      } break;
1061#if 0
1062      case 3: {
1063        int ix2 = iw + ido;
1064        radb3_ps(ido, l1, in, out, &wa[iw], &wa[ix2]);
1065      } break;
1066#endif
1067      case 2:
1068        radb2_ps(ido, l1, in, out, &wa[iw]);
1069        break;
1070      default:
1071        assert(0);
1072        break;
1073    }
1074    l1 = l2;
1075    iw += (ip - 1)*ido;
1076
1077    if (out == work2) {
1078      out = work1; in = work2;
1079    } else {
1080      out = work2; in = work1;
1081    }
1082  }
1083  return in; /* this is in fact the output .. */
1084}
1085
1086static int decompose(int n, int *ifac, const int *ntryh) {
1087  int nl = n, nf = 0, i, j = 0;
1088  for (j=0; ntryh[j]; ++j) {
1089    int ntry = ntryh[j];
1090    while (nl != 1) {
1091      int nq = nl / ntry;
1092      int nr = nl - ntry * nq;
1093      if (nr == 0) {
1094        ifac[2+nf++] = ntry;
1095        nl = nq;
1096        if (ntry == 2 && nf != 1) {
1097          for (i = 2; i <= nf; ++i) {
1098            int ib = nf - i + 2;
1099            ifac[ib + 1] = ifac[ib];
1100          }
1101          ifac[2] = 2;
1102        }
1103      } else break;
1104    }
1105  }
1106  ifac[0] = n;
1107  ifac[1] = nf;
1108  return nf;
1109}
1110
1111
1112
1113static void rffti1_ps(int n, float *wa, int *ifac)
1114{
1115  static const int ntryh[] = { 4,2,3,5,0 };
1116  int k1, j, ii;
1117
1118  int nf = decompose(n,ifac,ntryh);
1119  float argh = (float)((2*M_PI) / n);
1120  int is = 0;
1121  int nfm1 = nf - 1;
1122  int l1 = 1;
1123  for (k1 = 1; k1 <= nfm1; k1++) {
1124    int ip = ifac[k1 + 1];
1125    int ld = 0;
1126    int l2 = l1*ip;
1127    int ido = n / l2;
1128    int ipm = ip - 1;
1129    for (j = 1; j <= ipm; ++j) {
1130      float argld;
1131      int i = is, fi=0;
1132      ld += l1;
1133      argld = (float)ld*argh;
1134      for (ii = 3; ii <= ido; ii += 2) {
1135        i += 2;
1136        fi += 1;
1137        wa[i - 2] = cos((float)fi*argld);
1138        wa[i - 1] = sin((float)fi*argld);
1139      }
1140      is += ido;
1141    }
1142    l1 = l2;
1143  }
1144} /* rffti1 */
1145
1146static
1147void cffti1_ps(int n, float *wa, int *ifac)
1148{
1149  static const int ntryh[] = { 5,3,4,2,0 };
1150  int k1, j, ii;
1151
1152  int nf = decompose(n,ifac,ntryh);
1153  float argh = (float)((2*M_PI)/n);
1154  int i = 1;
1155  int l1 = 1;
1156  for (k1=1; k1<=nf; k1++) {
1157    int ip = ifac[k1+1];
1158    int ld = 0;
1159    int l2 = l1*ip;
1160    int ido = n / l2;
1161    int idot = ido + ido + 2;
1162    int ipm = ip - 1;
1163    for (j=1; j<=ipm; j++) {
1164      float argld;
1165      int i1 = i, fi = 0;
1166      wa[i-1] = 1;
1167      wa[i] = 0;
1168      ld += l1;
1169      argld = (float)ld*argh;
1170      for (ii = 4; ii <= idot; ii += 2) {
1171        i += 2;
1172        fi += 1;
1173        wa[i-1] = cos((float)fi*argld);
1174        wa[i] = sin((float)fi*argld);
1175      }
1176      if (ip > 5) {
1177        wa[i1-1] = wa[i-1];
1178        wa[i1] = wa[i];
1179      }
1180    }
1181    l1 = l2;
1182  }
1183} /* cffti1 */
1184
1185
1186static
1187v4sf *cfftf1_ps(int n, const v4sf *input_readonly, v4sf *work1, v4sf *work2, const float *wa, const int *ifac, int isign) {
1188  v4sf *in  = (v4sf*)input_readonly;
1189  v4sf *out = (in == work2 ? work1 : work2);
1190  int nf = ifac[1], k1;
1191  int l1 = 1;
1192  int iw = 0;
1193  assert(in != out && work1 != work2);
1194  for (k1=2; k1<=nf+1; k1++) {
1195    int ip = ifac[k1];
1196    int l2 = ip*l1;
1197    int ido = n / l2;
1198    int idot = ido + ido;
1199    switch (ip) {
1200#if 0
1201      case 5: {
1202        int ix2 = iw + idot;
1203        int ix3 = ix2 + idot;
1204        int ix4 = ix3 + idot;
1205        passf5_ps(idot, l1, in, out, &wa[iw], &wa[ix2], &wa[ix3], &wa[ix4], (float)isign);
1206      } break;
1207#endif
1208      case 4: {
1209        int ix2 = iw + idot;
1210        int ix3 = ix2 + idot;
1211        passf4_ps(idot, l1, in, out, &wa[iw], &wa[ix2], &wa[ix3], (float)isign);
1212      } break;
1213      case 2: {
1214        passf2_ps(idot, l1, in, out, &wa[iw], (float)isign);
1215      } break;
1216#if 0
1217      case 3: {
1218        int ix2 = iw + idot;
1219        passf3_ps(idot, l1, in, out, &wa[iw], &wa[ix2], (float)isign);
1220      } break;
1221#endif
1222      default:
1223        assert(0);
1224    }
1225    l1 = l2;
1226    iw += (ip - 1)*idot;
1227    if (out == work2) {
1228      out = work1; in = work2;
1229    } else {
1230      out = work2; in = work1;
1231    }
1232  }
1233
1234  return in; /* this is in fact the output .. */
1235}
1236
1237
1238struct PFFFT_Setup {
1239  int     N;
1240  int     Ncvec; /* nb of complex simd vectors (N/4 if PFFFT_COMPLEX, N/8 if PFFFT_REAL) */
1241  int ifac[15];
1242  pffft_transform_t transform;
1243  v4sf *data; /* allocated room for twiddle coefs */
1244  float *e;    /* points into 'data' , N/4*3 elements */
1245  float *twiddle; /* points into 'data', N/4 elements */
1246};
1247
1248static
1249PFFFT_Setup *pffft_new_setup(int N, pffft_transform_t transform) {
1250  PFFFT_Setup *s = (PFFFT_Setup*)malloc(sizeof(PFFFT_Setup));
1251  int k, m;
1252  if (!s) return s;
1253  /* unfortunately, the fft size must be a multiple of 16 for complex FFTs
1254     and 32 for real FFTs -- a lot of stuff would need to be rewritten to
1255     handle other cases (or maybe just switch to a scalar fft, I don't know..) */
1256  if (transform == PFFFT_REAL) { assert((N%(2*SIMD_SZ*SIMD_SZ))==0 && N>0); }
1257  if (transform == PFFFT_COMPLEX) { assert((N%(SIMD_SZ*SIMD_SZ))==0 && N>0); }
1258  /*assert((N % 32) == 0); */
1259  s->N = N;
1260  s->transform = transform;
1261  /* nb of complex simd vectors */
1262  s->Ncvec = (transform == PFFFT_REAL ? N/2 : N)/SIMD_SZ;
1263  s->data = (v4sf*)pffft_aligned_malloc(2*(size_t)s->Ncvec * sizeof(v4sf));
1264  if (!s->data) {free(s); return 0;}
1265  s->e = (float*)s->data;
1266  s->twiddle = (float*)(s->data + (2*s->Ncvec*(SIMD_SZ-1))/SIMD_SZ);
1267
1268  if (transform == PFFFT_REAL) {
1269    for (k=0; k < s->Ncvec; ++k) {
1270      int i = k/SIMD_SZ;
1271      int j = k%SIMD_SZ;
1272      for (m=0; m < SIMD_SZ-1; ++m) {
1273        float A = (float)(-2*M_PI*(m+1)*k / N);
1274        s->e[(2*(i*3 + m) + 0) * SIMD_SZ + j] = cos(A);
1275        s->e[(2*(i*3 + m) + 1) * SIMD_SZ + j] = sin(A);
1276      }
1277    }
1278    rffti1_ps(N/SIMD_SZ, s->twiddle, s->ifac);
1279  } else {
1280    for (k=0; k < s->Ncvec; ++k) {
1281      int i = k/SIMD_SZ;
1282      int j = k%SIMD_SZ;
1283      for (m=0; m < SIMD_SZ-1; ++m) {
1284        float A = (float)(-2*M_PI*(m+1)*k / N);
1285        s->e[(2*(i*3 + m) + 0)*SIMD_SZ + j] = cos(A);
1286        s->e[(2*(i*3 + m) + 1)*SIMD_SZ + j] = sin(A);
1287      }
1288    }
1289    cffti1_ps(N/SIMD_SZ, s->twiddle, s->ifac);
1290  }
1291
1292  /* check that N is decomposable with allowed prime factors */
1293  for (k=0, m=1; k < s->ifac[1]; ++k) { m *= s->ifac[2+k]; }
1294  if (m != N/SIMD_SZ) {
1295    pffft_destroy_setup(s); s = 0;
1296  }
1297
1298  return s;
1299}
1300
1301
1302static
1303void pffft_destroy_setup(PFFFT_Setup *s) {
1304  if (!s) return;
1305  pffft_aligned_free(s->data);
1306  free(s);
1307}
1308
1309#if !defined(PFFFT_SIMD_DISABLE)
1310
1311/* [0 0 1 2 3 4 5 6 7 8] -> [0 8 7 6 5 4 3 2 1] */
1312static void reversed_copy(int N, const v4sf *in, int in_stride, v4sf *out) {
1313  v4sf g0, g1;
1314  int k;
1315  INTERLEAVE2(in[0], in[1], g0, g1); in += in_stride;
1316
1317  *--out = VSWAPHL(g0, g1); /* [g0l, g0h], [g1l g1h] -> [g1l, g0h] */
1318  for (k=1; k < N; ++k) {
1319    v4sf h0, h1;
1320    INTERLEAVE2(in[0], in[1], h0, h1); in += in_stride;
1321    *--out = VSWAPHL(g1, h0);
1322    *--out = VSWAPHL(h0, h1);
1323    g1 = h1;
1324  }
1325  *--out = VSWAPHL(g1, g0);
1326}
1327
1328static void unreversed_copy(int N, const v4sf *in, v4sf *out, int out_stride) {
1329  v4sf g0, g1, h0, h1;
1330  int k;
1331  g0 = g1 = in[0]; ++in;
1332  for (k=1; k < N; ++k) {
1333    h0 = *in++; h1 = *in++;
1334    g1 = VSWAPHL(g1, h0);
1335    h0 = VSWAPHL(h0, h1);
1336    UNINTERLEAVE2(h0, g1, out[0], out[1]); out += out_stride;
1337    g1 = h1;
1338  }
1339  h0 = *in++; h1 = g0;
1340  g1 = VSWAPHL(g1, h0);
1341  h0 = VSWAPHL(h0, h1);
1342  UNINTERLEAVE2(h0, g1, out[0], out[1]);
1343}
1344
1345static
1346void pffft_zreorder(PFFFT_Setup *setup, const float *in, float *out, pffft_direction_t direction) {
1347  int k, N = setup->N, Ncvec = setup->Ncvec;
1348  const v4sf *vin = (const v4sf*)in;
1349  v4sf *vout = (v4sf*)out;
1350  assert(in != out);
1351  if (setup->transform == PFFFT_REAL) {
1352    int k, dk = N/32;
1353    if (direction == PFFFT_FORWARD) {
1354      for (k=0; k < dk; ++k) {
1355        INTERLEAVE2(vin[k*8 + 0], vin[k*8 + 1], vout[2*(0*dk + k) + 0], vout[2*(0*dk + k) + 1]);
1356        INTERLEAVE2(vin[k*8 + 4], vin[k*8 + 5], vout[2*(2*dk + k) + 0], vout[2*(2*dk + k) + 1]);
1357      }
1358      reversed_copy(dk, vin+2, 8, (v4sf*)(out + N/2));
1359      reversed_copy(dk, vin+6, 8, (v4sf*)(out + N));
1360    } else {
1361      for (k=0; k < dk; ++k) {
1362        UNINTERLEAVE2(vin[2*(0*dk + k) + 0], vin[2*(0*dk + k) + 1], vout[k*8 + 0], vout[k*8 + 1]);
1363        UNINTERLEAVE2(vin[2*(2*dk + k) + 0], vin[2*(2*dk + k) + 1], vout[k*8 + 4], vout[k*8 + 5]);
1364      }
1365      unreversed_copy(dk, (v4sf*)(in + N/4), (v4sf*)(out + N - 6*SIMD_SZ), -8);
1366      unreversed_copy(dk, (v4sf*)(in + 3*N/4), (v4sf*)(out + N - 2*SIMD_SZ), -8);
1367    }
1368  } else {
1369    if (direction == PFFFT_FORWARD) {
1370      for (k=0; k < Ncvec; ++k) {
1371        int kk = (k/4) + (k%4)*(Ncvec/4);
1372        INTERLEAVE2(vin[k*2], vin[k*2+1], vout[kk*2], vout[kk*2+1]);
1373      }
1374    } else {
1375      for (k=0; k < Ncvec; ++k) {
1376        int kk = (k/4) + (k%4)*(Ncvec/4);
1377        UNINTERLEAVE2(vin[kk*2], vin[kk*2+1], vout[k*2], vout[k*2+1]);
1378      }
1379    }
1380  }
1381}
1382
1383static
1384void pffft_cplx_finalize(int Ncvec, const v4sf *in, v4sf *out, const v4sf *e) {
1385  int k, dk = Ncvec/SIMD_SZ; /* number of 4x4 matrix blocks */
1386  v4sf r0, i0, r1, i1, r2, i2, r3, i3;
1387  v4sf sr0, dr0, sr1, dr1, si0, di0, si1, di1;
1388  assert(in != out);
1389  for (k=0; k < dk; ++k) {
1390    r0 = in[8*k+0]; i0 = in[8*k+1];
1391    r1 = in[8*k+2]; i1 = in[8*k+3];
1392    r2 = in[8*k+4]; i2 = in[8*k+5];
1393    r3 = in[8*k+6]; i3 = in[8*k+7];
1394    VTRANSPOSE4(r0,r1,r2,r3);
1395    VTRANSPOSE4(i0,i1,i2,i3);
1396    VCPLXMUL(r1,i1,e[k*6+0],e[k*6+1]);
1397    VCPLXMUL(r2,i2,e[k*6+2],e[k*6+3]);
1398    VCPLXMUL(r3,i3,e[k*6+4],e[k*6+5]);
1399
1400    sr0 = VADD(r0,r2); dr0 = VSUB(r0, r2);
1401    sr1 = VADD(r1,r3); dr1 = VSUB(r1, r3);
1402    si0 = VADD(i0,i2); di0 = VSUB(i0, i2);
1403    si1 = VADD(i1,i3); di1 = VSUB(i1, i3);
1404
1405    /*
1406      transformation for each column is:
1407
1408      [1   1   1   1   0   0   0   0]   [r0]
1409      [1   0  -1   0   0  -1   0   1]   [r1]
1410      [1  -1   1  -1   0   0   0   0]   [r2]
1411      [1   0  -1   0   0   1   0  -1]   [r3]
1412      [0   0   0   0   1   1   1   1] * [i0]
1413      [0   1   0  -1   1   0  -1   0]   [i1]
1414      [0   0   0   0   1  -1   1  -1]   [i2]
1415      [0  -1   0   1   1   0  -1   0]   [i3]
1416    */
1417
1418    r0 = VADD(sr0, sr1); i0 = VADD(si0, si1);
1419    r1 = VADD(dr0, di1); i1 = VSUB(di0, dr1);
1420    r2 = VSUB(sr0, sr1); i2 = VSUB(si0, si1);
1421    r3 = VSUB(dr0, di1); i3 = VADD(di0, dr1);
1422
1423    *out++ = r0; *out++ = i0; *out++ = r1; *out++ = i1;
1424    *out++ = r2; *out++ = i2; *out++ = r3; *out++ = i3;
1425  }
1426}
1427
1428static
1429void pffft_cplx_preprocess(int Ncvec, const v4sf *in, v4sf *out, const v4sf *e) {
1430  int k, dk = Ncvec/SIMD_SZ; /* number of 4x4 matrix blocks */
1431  v4sf r0, i0, r1, i1, r2, i2, r3, i3;
1432  v4sf sr0, dr0, sr1, dr1, si0, di0, si1, di1;
1433  assert(in != out);
1434  for (k=0; k < dk; ++k) {
1435    r0 = in[8*k+0]; i0 = in[8*k+1];
1436    r1 = in[8*k+2]; i1 = in[8*k+3];
1437    r2 = in[8*k+4]; i2 = in[8*k+5];
1438    r3 = in[8*k+6]; i3 = in[8*k+7];
1439
1440    sr0 = VADD(r0,r2); dr0 = VSUB(r0, r2);
1441    sr1 = VADD(r1,r3); dr1 = VSUB(r1, r3);
1442    si0 = VADD(i0,i2); di0 = VSUB(i0, i2);
1443    si1 = VADD(i1,i3); di1 = VSUB(i1, i3);
1444
1445    r0 = VADD(sr0, sr1); i0 = VADD(si0, si1);
1446    r1 = VSUB(dr0, di1); i1 = VADD(di0, dr1);
1447    r2 = VSUB(sr0, sr1); i2 = VSUB(si0, si1);
1448    r3 = VADD(dr0, di1); i3 = VSUB(di0, dr1);
1449
1450    VCPLXMULCONJ(r1,i1,e[k*6+0],e[k*6+1]);
1451    VCPLXMULCONJ(r2,i2,e[k*6+2],e[k*6+3]);
1452    VCPLXMULCONJ(r3,i3,e[k*6+4],e[k*6+5]);
1453
1454    VTRANSPOSE4(r0,r1,r2,r3);
1455    VTRANSPOSE4(i0,i1,i2,i3);
1456
1457    *out++ = r0; *out++ = i0; *out++ = r1; *out++ = i1;
1458    *out++ = r2; *out++ = i2; *out++ = r3; *out++ = i3;
1459  }
1460}
1461
1462
1463static ALWAYS_INLINE(void) pffft_real_finalize_4x4(const v4sf *in0, const v4sf *in1, const v4sf *in,
1464                            const v4sf *e, v4sf *out) {
1465  v4sf r0, i0, r1, i1, r2, i2, r3, i3;
1466  v4sf sr0, dr0, sr1, dr1, si0, di0, si1, di1;
1467  r0 = *in0; i0 = *in1;
1468  r1 = *in++; i1 = *in++; r2 = *in++; i2 = *in++; r3 = *in++; i3 = *in++;
1469  VTRANSPOSE4(r0,r1,r2,r3);
1470  VTRANSPOSE4(i0,i1,i2,i3);
1471
1472  /*
1473    transformation for each column is:
1474
1475    [1   1   1   1   0   0   0   0]   [r0]
1476    [1   0  -1   0   0  -1   0   1]   [r1]
1477    [1   0  -1   0   0   1   0  -1]   [r2]
1478    [1  -1   1  -1   0   0   0   0]   [r3]
1479    [0   0   0   0   1   1   1   1] * [i0]
1480    [0  -1   0   1  -1   0   1   0]   [i1]
1481    [0  -1   0   1   1   0  -1   0]   [i2]
1482    [0   0   0   0  -1   1  -1   1]   [i3]
1483  */
1484
1485  /*cerr << "matrix initial, before e , REAL:\n 1: " << r0 << "\n 1: " << r1 << "\n 1: " << r2 << "\n 1: " << r3 << "\n"; */
1486  /*cerr << "matrix initial, before e, IMAG :\n 1: " << i0 << "\n 1: " << i1 << "\n 1: " << i2 << "\n 1: " << i3 << "\n"; */
1487
1488  VCPLXMUL(r1,i1,e[0],e[1]);
1489  VCPLXMUL(r2,i2,e[2],e[3]);
1490  VCPLXMUL(r3,i3,e[4],e[5]);
1491
1492  /*cerr << "matrix initial, real part:\n 1: " << r0 << "\n 1: " << r1 << "\n 1: " << r2 << "\n 1: " << r3 << "\n"; */
1493  /*cerr << "matrix initial, imag part:\n 1: " << i0 << "\n 1: " << i1 << "\n 1: " << i2 << "\n 1: " << i3 << "\n"; */
1494
1495  sr0 = VADD(r0,r2); dr0 = VSUB(r0,r2);
1496  sr1 = VADD(r1,r3); dr1 = VSUB(r3,r1);
1497  si0 = VADD(i0,i2); di0 = VSUB(i0,i2);
1498  si1 = VADD(i1,i3); di1 = VSUB(i3,i1);
1499
1500  r0 = VADD(sr0, sr1);
1501  r3 = VSUB(sr0, sr1);
1502  i0 = VADD(si0, si1);
1503  i3 = VSUB(si1, si0);
1504  r1 = VADD(dr0, di1);
1505  r2 = VSUB(dr0, di1);
1506  i1 = VSUB(dr1, di0);
1507  i2 = VADD(dr1, di0);
1508
1509  *out++ = r0;
1510  *out++ = i0;
1511  *out++ = r1;
1512  *out++ = i1;
1513  *out++ = r2;
1514  *out++ = i2;
1515  *out++ = r3;
1516  *out++ = i3;
1517
1518}
1519
1520static NEVER_INLINE(void) pffft_real_finalize(int Ncvec, const v4sf *in, v4sf *out, const v4sf *e) {
1521  int k, dk = Ncvec/SIMD_SZ; /* number of 4x4 matrix blocks */
1522  /* fftpack order is f0r f1r f1i f2r f2i ... f(n-1)r f(n-1)i f(n)r */
1523
1524  v4sf_union cr, ci, *uout = (v4sf_union*)out;
1525  v4sf save = in[7], zero=VZERO();
1526  float xr0, xi0, xr1, xi1, xr2, xi2, xr3, xi3;
1527  static const float s = (float)(M_SQRT2/2);
1528
1529  cr.v = in[0]; ci.v = in[Ncvec*2-1];
1530  assert(in != out);
1531  pffft_real_finalize_4x4(&zero, &zero, in+1, e, out);
1532
1533  /*
1534    [cr0 cr1 cr2 cr3 ci0 ci1 ci2 ci3]
1535
1536    [Xr(1)]  ] [1   1   1   1   0   0   0   0]
1537    [Xr(N/4) ] [0   0   0   0   1   s   0  -s]
1538    [Xr(N/2) ] [1   0  -1   0   0   0   0   0]
1539    [Xr(3N/4)] [0   0   0   0   1  -s   0   s]
1540    [Xi(1)   ] [1  -1   1  -1   0   0   0   0]
1541    [Xi(N/4) ] [0   0   0   0   0  -s  -1  -s]
1542    [Xi(N/2) ] [0  -1   0   1   0   0   0   0]
1543    [Xi(3N/4)] [0   0   0   0   0  -s   1  -s]
1544  */
1545
1546  xr0=(cr.f[0]+cr.f[2]) + (cr.f[1]+cr.f[3]); uout[0].f[0] = xr0;
1547  xi0=(cr.f[0]+cr.f[2]) - (cr.f[1]+cr.f[3]); uout[1].f[0] = xi0;
1548  xr2=(cr.f[0]-cr.f[2]);                     uout[4].f[0] = xr2;
1549  xi2=(cr.f[3]-cr.f[1]);                     uout[5].f[0] = xi2;
1550  xr1= ci.f[0] + s*(ci.f[1]-ci.f[3]);        uout[2].f[0] = xr1;
1551  xi1=-ci.f[2] - s*(ci.f[1]+ci.f[3]);        uout[3].f[0] = xi1;
1552  xr3= ci.f[0] - s*(ci.f[1]-ci.f[3]);        uout[6].f[0] = xr3;
1553  xi3= ci.f[2] - s*(ci.f[1]+ci.f[3]);        uout[7].f[0] = xi3;
1554
1555  for (k=1; k < dk; ++k) {
1556    v4sf save_next = in[8*k+7];
1557    pffft_real_finalize_4x4(&save, &in[8*k+0], in + 8*k+1,
1558                           e + k*6, out + k*8);
1559    save = save_next;
1560  }
1561
1562}
1563
1564static ALWAYS_INLINE(void) pffft_real_preprocess_4x4(const v4sf *in,
1565                                             const v4sf *e, v4sf *out, int first) {
1566  v4sf r0=in[0], i0=in[1], r1=in[2], i1=in[3], r2=in[4], i2=in[5], r3=in[6], i3=in[7];
1567  /*
1568    transformation for each column is:
1569
1570    [1   1   1   1   0   0   0   0]   [r0]
1571    [1   0   0  -1   0  -1  -1   0]   [r1]
1572    [1  -1  -1   1   0   0   0   0]   [r2]
1573    [1   0   0  -1   0   1   1   0]   [r3]
1574    [0   0   0   0   1  -1   1  -1] * [i0]
1575    [0  -1   1   0   1   0   0   1]   [i1]
1576    [0   0   0   0   1   1  -1  -1]   [i2]
1577    [0   1  -1   0   1   0   0   1]   [i3]
1578  */
1579
1580  v4sf sr0 = VADD(r0,r3), dr0 = VSUB(r0,r3);
1581  v4sf sr1 = VADD(r1,r2), dr1 = VSUB(r1,r2);
1582  v4sf si0 = VADD(i0,i3), di0 = VSUB(i0,i3);
1583  v4sf si1 = VADD(i1,i2), di1 = VSUB(i1,i2);
1584
1585  r0 = VADD(sr0, sr1);
1586  r2 = VSUB(sr0, sr1);
1587  r1 = VSUB(dr0, si1);
1588  r3 = VADD(dr0, si1);
1589  i0 = VSUB(di0, di1);
1590  i2 = VADD(di0, di1);
1591  i1 = VSUB(si0, dr1);
1592  i3 = VADD(si0, dr1);
1593
1594  VCPLXMULCONJ(r1,i1,e[0],e[1]);
1595  VCPLXMULCONJ(r2,i2,e[2],e[3]);
1596  VCPLXMULCONJ(r3,i3,e[4],e[5]);
1597
1598  VTRANSPOSE4(r0,r1,r2,r3);
1599  VTRANSPOSE4(i0,i1,i2,i3);
1600
1601  if (!first) {
1602    *out++ = r0;
1603    *out++ = i0;
1604  }
1605  *out++ = r1;
1606  *out++ = i1;
1607  *out++ = r2;
1608  *out++ = i2;
1609  *out++ = r3;
1610  *out++ = i3;
1611}
1612
1613static NEVER_INLINE(void) pffft_real_preprocess(int Ncvec, const v4sf *in, v4sf *out, const v4sf *e) {
1614  int k, dk = Ncvec/SIMD_SZ; /* number of 4x4 matrix blocks */
1615  /* fftpack order is f0r f1r f1i f2r f2i ... f(n-1)r f(n-1)i f(n)r */
1616
1617  v4sf_union Xr, Xi, *uout = (v4sf_union*)out;
1618  float cr0, ci0, cr1, ci1, cr2, ci2, cr3, ci3;
1619  static const float s = (float)M_SQRT2;
1620  assert(in != out);
1621  for (k=0; k < 4; ++k) {
1622    Xr.f[k] = ((float*)in)[8*k];
1623    Xi.f[k] = ((float*)in)[8*k+4];
1624  }
1625
1626  pffft_real_preprocess_4x4(in, e, out+1, 1); /* will write only 6 values */
1627
1628  /*
1629    [Xr0 Xr1 Xr2 Xr3 Xi0 Xi1 Xi2 Xi3]
1630
1631    [cr0] [1   0   2   0   1   0   0   0]
1632    [cr1] [1   0   0   0  -1   0  -2   0]
1633    [cr2] [1   0  -2   0   1   0   0   0]
1634    [cr3] [1   0   0   0  -1   0   2   0]
1635    [ci0] [0   2   0   2   0   0   0   0]
1636    [ci1] [0   s   0  -s   0  -s   0  -s]
1637    [ci2] [0   0   0   0   0  -2   0   2]
1638    [ci3] [0  -s   0   s   0  -s   0  -s]
1639  */
1640  for (k=1; k < dk; ++k) {
1641    pffft_real_preprocess_4x4(in+8*k, e + k*6, out-1+k*8, 0);
1642  }
1643
1644  cr0=(Xr.f[0]+Xi.f[0]) + 2*Xr.f[2]; uout[0].f[0] = cr0;
1645  cr1=(Xr.f[0]-Xi.f[0]) - 2*Xi.f[2]; uout[0].f[1] = cr1;
1646  cr2=(Xr.f[0]+Xi.f[0]) - 2*Xr.f[2]; uout[0].f[2] = cr2;
1647  cr3=(Xr.f[0]-Xi.f[0]) + 2*Xi.f[2]; uout[0].f[3] = cr3;
1648  ci0= 2*(Xr.f[1]+Xr.f[3]);                       uout[2*Ncvec-1].f[0] = ci0;
1649  ci1= s*(Xr.f[1]-Xr.f[3]) - s*(Xi.f[1]+Xi.f[3]); uout[2*Ncvec-1].f[1] = ci1;
1650  ci2= 2*(Xi.f[3]-Xi.f[1]);                       uout[2*Ncvec-1].f[2] = ci2;
1651  ci3=-s*(Xr.f[1]-Xr.f[3]) - s*(Xi.f[1]+Xi.f[3]); uout[2*Ncvec-1].f[3] = ci3;
1652}
1653
1654
1655static
1656void pffft_transform_internal(PFFFT_Setup *setup, const float *finput, float *foutput, v4sf *scratch,
1657                             pffft_direction_t direction, int ordered) {
1658  int k, Ncvec   = setup->Ncvec;
1659  int nf_odd = (setup->ifac[1] & 1);
1660
1661#if 0
1662  /* temporary buffer is allocated on the stack if the scratch pointer is NULL */
1663  int stack_allocate = (scratch == 0 ? Ncvec*2 : 1);
1664  VLA_ARRAY_ON_STACK(v4sf, scratch_on_stack, stack_allocate);
1665#endif
1666
1667  const v4sf *vinput = (const v4sf*)finput;
1668  v4sf *voutput      = (v4sf*)foutput;
1669  v4sf *buff[2];
1670  int ib = (nf_odd ^ ordered ? 1 : 0);
1671  buff[0] = voutput; buff[1] = scratch;
1672
1673  assert(VALIGNED(finput) && VALIGNED(foutput));
1674
1675  /*assert(finput != foutput); */
1676  if (direction == PFFFT_FORWARD) {
1677    ib = !ib;
1678    if (setup->transform == PFFFT_REAL) {
1679      ib = (rfftf1_ps(Ncvec*2, vinput, buff[ib], buff[!ib],
1680                      setup->twiddle, &setup->ifac[0]) == buff[0] ? 0 : 1);
1681      pffft_real_finalize(Ncvec, buff[ib], buff[!ib], (v4sf*)setup->e);
1682    } else {
1683      v4sf *tmp = buff[ib];
1684      for (k=0; k < Ncvec; ++k) {
1685        UNINTERLEAVE2(vinput[k*2], vinput[k*2+1], tmp[k*2], tmp[k*2+1]);
1686      }
1687      ib = (cfftf1_ps(Ncvec, buff[ib], buff[!ib], buff[ib],
1688                      setup->twiddle, &setup->ifac[0], -1) == buff[0] ? 0 : 1);
1689      pffft_cplx_finalize(Ncvec, buff[ib], buff[!ib], (v4sf*)setup->e);
1690    }
1691    if (ordered) {
1692      pffft_zreorder(setup, (float*)buff[!ib], (float*)buff[ib], PFFFT_FORWARD);
1693    } else ib = !ib;
1694  } else {
1695    if (vinput == buff[ib]) {
1696      ib = !ib; /* may happen when finput == foutput */
1697    }
1698    if (ordered) {
1699      pffft_zreorder(setup, (float*)vinput, (float*)buff[ib], PFFFT_BACKWARD);
1700      vinput = buff[ib]; ib = !ib;
1701    }
1702    if (setup->transform == PFFFT_REAL) {
1703      pffft_real_preprocess(Ncvec, vinput, buff[ib], (v4sf*)setup->e);
1704      ib = (rfftb1_ps(Ncvec*2, buff[ib], buff[0], buff[1],
1705                      setup->twiddle, &setup->ifac[0]) == buff[0] ? 0 : 1);
1706    } else {
1707      pffft_cplx_preprocess(Ncvec, vinput, buff[ib], (v4sf*)setup->e);
1708      ib = (cfftf1_ps(Ncvec, buff[ib], buff[0], buff[1],
1709                      setup->twiddle, &setup->ifac[0], +1) == buff[0] ? 0 : 1);
1710      for (k=0; k < Ncvec; ++k) {
1711        INTERLEAVE2(buff[ib][k*2], buff[ib][k*2+1], buff[ib][k*2], buff[ib][k*2+1]);
1712      }
1713    }
1714  }
1715
1716  if (buff[ib] != voutput) {
1717    /* extra copy required -- this situation should only happen when finput == foutput */
1718    assert(finput==foutput);
1719    for (k=0; k < Ncvec; ++k) {
1720      v4sf a = buff[ib][2*k], b = buff[ib][2*k+1];
1721      voutput[2*k] = a; voutput[2*k+1] = b;
1722    }
1723    ib = !ib;
1724  }
1725  assert(buff[ib] == voutput);
1726}
1727
1728#if 0
1729void pffft_zconvolve_accumulate(PFFFT_Setup *s, const float *a, const float *b, float *ab, float scaling) {
1730  int Ncvec = s->Ncvec;
1731  const v4sf * RESTRICT va = (const v4sf*)a;
1732  const v4sf * RESTRICT vb = (const v4sf*)b;
1733  v4sf * RESTRICT vab = (v4sf*)ab;
1734
1735#ifdef __arm__
1736  __builtin_prefetch(va);
1737  __builtin_prefetch(vb);
1738  __builtin_prefetch(vab);
1739  __builtin_prefetch(va+2);
1740  __builtin_prefetch(vb+2);
1741  __builtin_prefetch(vab+2);
1742  __builtin_prefetch(va+4);
1743  __builtin_prefetch(vb+4);
1744  __builtin_prefetch(vab+4);
1745  __builtin_prefetch(va+6);
1746  __builtin_prefetch(vb+6);
1747  __builtin_prefetch(vab+6);
1748# ifndef __clang__
1749#   define ZCONVOLVE_USING_INLINE_NEON_ASM
1750# endif
1751#endif
1752
1753  float ar, ai, br, bi, abr, abi;
1754#ifndef ZCONVOLVE_USING_INLINE_ASM
1755  v4sf vscal = LD_PS1(scaling);
1756  int i;
1757#endif
1758
1759  assert(VALIGNED(a) && VALIGNED(b) && VALIGNED(ab));
1760  ar = ((v4sf_union*)va)[0].f[0];
1761  ai = ((v4sf_union*)va)[1].f[0];
1762  br = ((v4sf_union*)vb)[0].f[0];
1763  bi = ((v4sf_union*)vb)[1].f[0];
1764  abr = ((v4sf_union*)vab)[0].f[0];
1765  abi = ((v4sf_union*)vab)[1].f[0];
1766
1767#ifdef ZCONVOLVE_USING_INLINE_ASM /* inline asm version, unfortunately miscompiled by clang 3.2, at least on ubuntu.. so this will be restricted to gcc */
1768  const float *a_ = a, *b_ = b; float *ab_ = ab;
1769  int N = Ncvec;
1770  asm volatile("mov         r8, %2                  \n"
1771               "vdup.f32    q15, %4                 \n"
1772               "1:                                  \n"
1773               "pld         [%0,#64]                \n"
1774               "pld         [%1,#64]                \n"
1775               "pld         [%2,#64]                \n"
1776               "pld         [%0,#96]                \n"
1777               "pld         [%1,#96]                \n"
1778               "pld         [%2,#96]                \n"
1779               "vld1.f32    {q0,q1},   [%0,:128]!         \n"
1780               "vld1.f32    {q4,q5},   [%1,:128]!         \n"
1781               "vld1.f32    {q2,q3},   [%0,:128]!         \n"
1782               "vld1.f32    {q6,q7},   [%1,:128]!         \n"
1783               "vld1.f32    {q8,q9},   [r8,:128]!          \n"
1784
1785               "vmul.f32    q10, q0, q4             \n"
1786               "vmul.f32    q11, q0, q5             \n"
1787               "vmul.f32    q12, q2, q6             \n"
1788               "vmul.f32    q13, q2, q7             \n"
1789               "vmls.f32    q10, q1, q5             \n"
1790               "vmla.f32    q11, q1, q4             \n"
1791               "vld1.f32    {q0,q1}, [r8,:128]!     \n"
1792               "vmls.f32    q12, q3, q7             \n"
1793               "vmla.f32    q13, q3, q6             \n"
1794               "vmla.f32    q8, q10, q15            \n"
1795               "vmla.f32    q9, q11, q15            \n"
1796               "vmla.f32    q0, q12, q15            \n"
1797               "vmla.f32    q1, q13, q15            \n"
1798               "vst1.f32    {q8,q9},[%2,:128]!    \n"
1799               "vst1.f32    {q0,q1},[%2,:128]!    \n"
1800               "subs        %3, #2                  \n"
1801               "bne         1b                      \n"
1802               : "+r"(a_), "+r"(b_), "+r"(ab_), "+r"(N) : "r"(scaling) : "r8", "q0","q1","q2","q3","q4","q5","q6","q7","q8","q9", "q10","q11","q12","q13","q15","memory");
1803#else /* default routine, works fine for non-arm cpus with current compilers */
1804  for (i=0; i < Ncvec; i += 2) {
1805    v4sf ar, ai, br, bi;
1806    ar = va[2*i+0]; ai = va[2*i+1];
1807    br = vb[2*i+0]; bi = vb[2*i+1];
1808    VCPLXMUL(ar, ai, br, bi);
1809    vab[2*i+0] = VMADD(ar, vscal, vab[2*i+0]);
1810    vab[2*i+1] = VMADD(ai, vscal, vab[2*i+1]);
1811    ar = va[2*i+2]; ai = va[2*i+3];
1812    br = vb[2*i+2]; bi = vb[2*i+3];
1813    VCPLXMUL(ar, ai, br, bi);
1814    vab[2*i+2] = VMADD(ar, vscal, vab[2*i+2]);
1815    vab[2*i+3] = VMADD(ai, vscal, vab[2*i+3]);
1816  }
1817#endif
1818  if (s->transform == PFFFT_REAL) {
1819    ((v4sf_union*)vab)[0].f[0] = abr + ar*br*scaling;
1820    ((v4sf_union*)vab)[1].f[0] = abi + ai*bi*scaling;
1821  }
1822}
1823#endif
1824
1825
1826#else /* defined(PFFFT_SIMD_DISABLE) */
1827
1828/* standard routine using scalar floats, without SIMD stuff. */
1829
1830#define pffft_zreorder_nosimd pffft_zreorder
1831static
1832void pffft_zreorder_nosimd(PFFFT_Setup *setup, const float *in, float *out, pffft_direction_t direction) {
1833  int k, N = setup->N;
1834  if (setup->transform == PFFFT_COMPLEX) {
1835    for (k=0; k < 2*N; ++k) out[k] = in[k];
1836    return;
1837  }
1838  else if (direction == PFFFT_FORWARD) {
1839    float x_N = in[N-1];
1840    for (k=N-1; k > 1; --k) out[k] = in[k-1];
1841    out[0] = in[0];
1842    out[1] = x_N;
1843  } else {
1844    float x_N = in[1];
1845    for (k=1; k < N-1; ++k) out[k] = in[k+1];
1846    out[0] = in[0];
1847    out[N-1] = x_N;
1848  }
1849}
1850
1851#define pffft_transform_internal_nosimd pffft_transform_internal
1852static
1853void pffft_transform_internal_nosimd(PFFFT_Setup *setup, const float *input, float *output, float *scratch,
1854                                    pffft_direction_t direction, int ordered) {
1855  int Ncvec   = setup->Ncvec;
1856  int nf_odd = (setup->ifac[1] & 1);
1857
1858#if 0
1859  /* temporary buffer is allocated on the stack if the scratch pointer is NULL */
1860  int stack_allocate = (scratch == 0 ? Ncvec*2 : 1);
1861  VLA_ARRAY_ON_STACK(v4sf, scratch_on_stack, stack_allocate);
1862#endif
1863  float *buff[2];
1864  int ib;
1865  /* if (scratch == 0) scratch = scratch_on_stack; */
1866  buff[0] = output; buff[1] = scratch;
1867
1868  if (setup->transform == PFFFT_COMPLEX) ordered = 0; /* it is always ordered. */
1869  ib = (nf_odd ^ ordered ? 1 : 0);
1870
1871  if (direction == PFFFT_FORWARD) {
1872    if (setup->transform == PFFFT_REAL) {
1873      ib = (rfftf1_ps(Ncvec*2, input, buff[ib], buff[!ib],
1874                      setup->twiddle, &setup->ifac[0]) == buff[0] ? 0 : 1);
1875    } else {
1876      ib = (cfftf1_ps(Ncvec, input, buff[ib], buff[!ib],
1877                      setup->twiddle, &setup->ifac[0], -1) == buff[0] ? 0 : 1);
1878    }
1879    if (ordered) {
1880      pffft_zreorder(setup, buff[ib], buff[!ib], PFFFT_FORWARD); ib = !ib;
1881    }
1882  } else {
1883    if (input == buff[ib]) {
1884      ib = !ib; /* may happen when finput == foutput */
1885    }
1886    if (ordered) {
1887      pffft_zreorder(setup, input, buff[!ib], PFFFT_BACKWARD);
1888      input = buff[!ib];
1889    }
1890    if (setup->transform == PFFFT_REAL) {
1891      ib = (rfftb1_ps(Ncvec*2, input, buff[ib], buff[!ib],
1892                      setup->twiddle, &setup->ifac[0]) == buff[0] ? 0 : 1);
1893    } else {
1894      ib = (cfftf1_ps(Ncvec, input, buff[ib], buff[!ib],
1895                      setup->twiddle, &setup->ifac[0], +1) == buff[0] ? 0 : 1);
1896    }
1897  }
1898  if (buff[ib] != output) {
1899    int k;
1900    /* extra copy required -- this situation should happens only when finput == foutput */
1901    assert(input==output);
1902    for (k=0; k < Ncvec; ++k) {
1903      float a = buff[ib][2*k], b = buff[ib][2*k+1];
1904      output[2*k] = a; output[2*k+1] = b;
1905    }
1906    ib = !ib;
1907  }
1908  assert(buff[ib] == output);
1909}
1910
1911#if 0
1912#define pffft_zconvolve_accumulate_nosimd pffft_zconvolve_accumulate
1913void pffft_zconvolve_accumulate_nosimd(PFFFT_Setup *s, const float *a, const float *b,
1914                                       float *ab, float scaling) {
1915  int i, Ncvec = s->Ncvec;
1916
1917  if (s->transform == PFFFT_REAL) {
1918    /* take care of the fftpack ordering */
1919    ab[0] += a[0]*b[0]*scaling;
1920    ab[2*Ncvec-1] += a[2*Ncvec-1]*b[2*Ncvec-1]*scaling;
1921    ++ab; ++a; ++b; --Ncvec;
1922  }
1923  for (i=0; i < Ncvec; ++i) {
1924    float ar, ai, br, bi;
1925    ar = a[2*i+0]; ai = a[2*i+1];
1926    br = b[2*i+0]; bi = b[2*i+1];
1927    VCPLXMUL(ar, ai, br, bi);
1928    ab[2*i+0] += ar*scaling;
1929    ab[2*i+1] += ai*scaling;
1930  }
1931}
1932#endif
1933
1934#endif /* defined(PFFFT_SIMD_DISABLE) */
1935
1936static
1937void pffft_transform(PFFFT_Setup *setup, const float *input, float *output, float *work, pffft_direction_t direction) {
1938  pffft_transform_internal(setup, input, output, (v4sf*)work, direction, 0);
1939}
1940
1941static
1942void pffft_transform_ordered(PFFFT_Setup *setup, const float *input, float *output, float *work, pffft_direction_t direction) {
1943  pffft_transform_internal(setup, input, output, (v4sf*)work, direction, 1);
1944}
1945
1946#endif