fork/rust-sdks
git clone https://git.julia.blue/fork/rust-sdks
2e4d385
main
1/* https://bitbucket.org/jpommier/pffft/raw/483453d8f7661058e74aa4e7cf5c27bcd7887e7a/pffft.c 2* with minor changes for libsoxr. */ 3 4/* Copyright (c) 2013 Julien Pommier ( pommier@modartt.com ) 5 6Based on original fortran 77 code from FFTPACKv4 from NETLIB 7(http://www.netlib.org/fftpack), authored by Dr Paul Swarztrauber 8of NCAR, in 1985. 9 10As confirmed by the NCAR fftpack software curators, the following 11FFTPACKv5 license applies to FFTPACKv4 sources. My changes are 12released under the same terms. 13 14FFTPACK license: 15 16http://www.cisl.ucar.edu/css/software/fftpack5/ftpk.html 17 18Copyright (c) 2004 the University Corporation for Atmospheric 19Research ("UCAR"). All rights reserved. Developed by NCAR's 20Computational and Information Systems Laboratory, UCAR, 21www.cisl.ucar.edu. 22 23Redistribution and use of the Software in source and binary forms, 24with or without modification, is permitted provided that the 25following conditions are met: 26 27- Neither the names of NCAR's Computational and Information Systems 28Laboratory, the University Corporation for Atmospheric Research, 29nor the names of its sponsors or contributors may be used to 30endorse or promote products derived from this Software without 31specific prior written permission. 32 33- Redistributions of source code must retain the above copyright 34notices, this list of conditions, and the disclaimer below. 35 36- Redistributions in binary form must reproduce the above copyright 37notice, this list of conditions, and the disclaimer below in the 38documentation and/or other materials provided with the 39distribution. 40 41THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 42EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO THE WARRANTIES OF 43MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 44NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT 45HOLDERS BE LIABLE FOR ANY CLAIM, INDIRECT, INCIDENTAL, SPECIAL, 46EXEMPLARY, OR CONSEQUENTIAL DAMAGES OR OTHER LIABILITY, WHETHER IN AN 47ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 48CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE 49SOFTWARE. 50 51 52PFFFT : a Pretty Fast FFT. 53 54This file is largerly based on the original FFTPACK implementation, modified in 55order to take advantage of SIMD instructions of modern CPUs. 56*/ 57 58/* 59ChangeLog: 60- 2011/10/02, version 1: This is the very first release of this file. 61*/ 62 63#include "pffft.h" 64#include <stdlib.h> 65#include <stdio.h> 66#include <math.h> 67#include <assert.h> 68 69/* detect compiler flavour */ 70#if defined(_MSC_VER ) 71# define COMPILER_MSVC 72#elif defined(__GNUC__ ) 73# define COMPILER_GCC 74#endif 75 76#if defined(COMPILER_GCC ) 77# define ALWAYS_INLINE (return_type ) inline return_type __attribute__ ((always_inline)) 78# define NEVER_INLINE (return_type ) return_type __attribute__ ((noinline)) 79# define RESTRICT __restrict 80# define VLA_ARRAY_ON_STACK (type__ ,varname__ ,size__ ) type__ varname__[size__]; 81#elif defined(COMPILER_MSVC ) 82# define ALWAYS_INLINE (return_type ) __forceinline return_type 83# define NEVER_INLINE (return_type ) __declspec(noinline) return_type 84# define RESTRICT __restrict 85# define VLA_ARRAY_ON_STACK (type__ ,varname__ ,size__ ) type__ *varname__ = (type__*)_alloca(size__ * sizeof(type__)) 86#endif 87 88 89/* 90vector support macros: the rest of the code is independant of 91SSE/Altivec/NEON -- adding support for other platforms with 4-element 92vectors should be limited to these macros 93*/ 94 95 96/* define PFFFT_SIMD_DISABLE if you want to use scalar code instead of simd code */ 97/*#define PFFFT_SIMD_DISABLE */ 98 99/* 100Altivec support macros 101*/ 102#if !defined(PFFFT_SIMD_DISABLE )&& (defined(__ppc__ )|| defined(__ppc64__ )) 103typedef vector float v4sf ; 104# define SIMD_SZ 4 105# define VZERO () ((vector float) vec_splat_u8(0)) 106# define VMUL (a ,b ) vec_madd(a,b, VZERO()) 107# define VADD (a ,b ) vec_add(a,b) 108# define VMADD (a ,b ,c ) vec_madd(a,b,c) 109# define VSUB (a ,b ) vec_sub(a,b) 110inline v4sf ld_ps1 (const float * p ) {v4sf v = vec_lde (0 ,p );return vec_splat (vec_perm (v ,v ,vec_lvsl (0 ,p )),0 ); } 111# define LD_PS1 (p ) ld_ps1(&p) 112# define INTERLEAVE2 (in1 ,in2 ,out1 ,out2 ) { v4sf tmp__ = vec_mergeh(in1, in2); out2 = vec_mergel(in1, in2); out1 = tmp__; } 113# define UNINTERLEAVE2 (in1 ,in2 ,out1 ,out2 ) { \ 114 vector unsigned char vperm1 = (vector unsigned char)(0,1,2,3,8,9,10,11,16,17,18,19,24,25,26,27); \ 115 vector unsigned char vperm2 = (vector unsigned char)(4,5,6,7,12,13,14,15,20,21,22,23,28,29,30,31); \ 116 v4sf tmp__ = vec_perm(in1, in2, vperm1); out2 = vec_perm(in1, in2, vperm2); out1 = tmp__; \ 117 } 118# define VTRANSPOSE4 (x0 ,x1 ,x2 ,x3 ) { \ 119 v4sf y0 = vec_mergeh(x0, x2); \ 120 v4sf y1 = vec_mergel(x0, x2); \ 121 v4sf y2 = vec_mergeh(x1, x3); \ 122 v4sf y3 = vec_mergel(x1, x3); \ 123 x0 = vec_mergeh(y0, y2); \ 124 x1 = vec_mergel(y0, y2); \ 125 x2 = vec_mergeh(y1, y3); \ 126 x3 = vec_mergel(y1, y3); \ 127 } 128# define VSWAPHL (a ,b ) vec_perm(a,b, (vector unsigned char)(16,17,18,19,20,21,22,23,8,9,10,11,12,13,14,15)) 129# define VALIGNED (ptr ) ((((long)(ptr)) & 0xF) == 0) 130 131/* 132SSE1 support macros 133*/ 134#elif !defined(PFFFT_SIMD_DISABLE )&& (defined(__x86_64__ )|| defined(_M_X64 )|| defined(i386 )|| defined(_M_IX86 )) 135 136# define SIMD_SZ 4/* 4 floats by simd vector -- this is pretty much hardcoded in the preprocess/finalize functions anyway so you will have to work if you want to enable AVX with its 256-bit vectors. */ 137 138#if !PFFFT_DOUBLE 139#include <xmmintrin.h> 140typedef __m128 v4sf ; 141# define VZERO () _mm_setzero_ps() 142# define VMUL (a ,b ) _mm_mul_ps(a,b) 143# define VADD (a ,b ) _mm_add_ps(a,b) 144# define VMADD (a ,b ,c ) _mm_add_ps(_mm_mul_ps(a,b), c) 145# define VSUB (a ,b ) _mm_sub_ps(a,b) 146# define LD_PS1 (p ) _mm_set1_ps(p) 147# define INTERLEAVE2 (in1 ,in2 ,out1 ,out2 ) { v4sf tmp__ = _mm_unpacklo_ps(in1, in2); out2 = _mm_unpackhi_ps(in1, in2); out1 = tmp__; } 148# define UNINTERLEAVE2 (in1 ,in2 ,out1 ,out2 ) { v4sf tmp__ = _mm_shuffle_ps(in1, in2, _MM_SHUFFLE(2,0,2,0)); out2 = _mm_shuffle_ps(in1, in2, _MM_SHUFFLE(3,1,3,1)); out1 = tmp__; } 149# define VTRANSPOSE4 (x0 ,x1 ,x2 ,x3 ) _MM_TRANSPOSE4_PS(x0,x1,x2,x3) 150# define VSWAPHL (a ,b ) _mm_shuffle_ps(b, a, _MM_SHUFFLE(3,2,1,0)) 151# define VALIGNED (ptr ) ((((long)(ptr)) & 0xF) == 0) 152 153#else 154#include "pffft-avx.h" 155#endif 156 157/* 158ARM NEON support macros 159*/ 160#elif !defined(PFFFT_SIMD_DISABLE )&& defined(__arm__ ) 161# include <arm_neon.h> 162typedef float32x4_t v4sf ; 163# define SIMD_SZ 4 164# define VZERO () vdupq_n_f32(0) 165# define VMUL (a ,b ) vmulq_f32(a,b) 166# define VADD (a ,b ) vaddq_f32(a,b) 167# define VMADD (a ,b ,c ) vmlaq_f32(c,a,b) 168# define VSUB (a ,b ) vsubq_f32(a,b) 169# define LD_PS1 (p ) vld1q_dup_f32(&(p)) 170# define INTERLEAVE2 (in1 ,in2 ,out1 ,out2 ) { float32x4x2_t tmp__ = vzipq_f32(in1,in2); out1=tmp__.val[0]; out2=tmp__.val[1]; } 171# define UNINTERLEAVE2 (in1 ,in2 ,out1 ,out2 ) { float32x4x2_t tmp__ = vuzpq_f32(in1,in2); out1=tmp__.val[0]; out2=tmp__.val[1]; } 172# define VTRANSPOSE4 (x0 ,x1 ,x2 ,x3 ) { \ 173 float32x4x2_t t0_ = vzipq_f32(x0, x2); \ 174 float32x4x2_t t1_ = vzipq_f32(x1, x3); \ 175 float32x4x2_t u0_ = vzipq_f32(t0_.val[0], t1_.val[0]); \ 176 float32x4x2_t u1_ = vzipq_f32(t0_.val[1], t1_.val[1]); \ 177 x0 = u0_.val[0]; x1 = u0_.val[1]; x2 = u1_.val[0]; x3 = u1_.val[1]; \ 178 } 179/* marginally faster version */ 180/*# define VTRANSPOSE4(x0,x1,x2,x3) { asm("vtrn.32 %q0, %q1;\n vtrn.32 %q2,%q3\n vswp %f0,%e2\n vswp %f1,%e3" : "+w"(x0), "+w"(x1), "+w"(x2), "+w"(x3)::); } */ 181# define VSWAPHL (a ,b ) vcombine_f32(vget_low_f32(b), vget_high_f32(a)) 182# define VALIGNED (ptr ) ((((long)(ptr)) & 0x3) == 0) 183#else 184# if !defined(PFFFT_SIMD_DISABLE ) 185# warning "building with simd disabled !\n"; 186# define PFFFT_SIMD_DISABLE /* fallback to scalar code */ 187# endif 188#endif 189 190#if PFFFT_DOUBLE 191#define float double 192#endif 193 194/* fallback mode for situations where SSE/Altivec are not available, use scalar mode instead */ 195#ifdef PFFFT_SIMD_DISABLE 196typedef float v4sf ; 197# define SIMD_SZ 1 198# define VZERO () 0.f 199# define VMUL (a ,b ) ((a)*(b)) 200# define VADD (a ,b ) ((a)+(b)) 201# define VMADD (a ,b ,c ) ((a)*(b)+(c)) 202# define VSUB (a ,b ) ((a)-(b)) 203# define LD_PS1 (p ) (p) 204# define VALIGNED (ptr ) ((((long)(ptr)) & 0x3) == 0) 205#endif 206 207/* shortcuts for complex multiplcations */ 208#define VCPLXMUL (ar ,ai ,br ,bi ) { v4sf tmp; tmp=VMUL(ar,bi); ar=VMUL(ar,br); ar=VSUB(ar,VMUL(ai,bi)); ai=VMUL(ai,br); ai=VADD(ai,tmp); } 209#define VCPLXMULCONJ (ar ,ai ,br ,bi ) { v4sf tmp; tmp=VMUL(ar,bi); ar=VMUL(ar,br); ar=VADD(ar,VMUL(ai,bi)); ai=VMUL(ai,br); ai=VSUB(ai,tmp); } 210#ifndef SVMUL 211/* multiply a scalar with a vector */ 212#define SVMUL (f ,v ) VMUL(LD_PS1(f),v) 213#endif 214 215#if !definedPFFT_MACROS_ONLY 216 217#if !defined(PFFFT_SIMD_DISABLE ) 218typedef union v4sf_union { 219v4sf v ; 220float f [4 ]; 221}v4sf_union ; 222 223#if 0 224#include <string.h> 225 226#define assertv4 (v ,f0 ,f1 ,f2 ,f3 ) assert(v.f[0] == (f0) && v.f[1] == (f1) && v.f[2] == (f2) && v.f[3] == (f3)) 227 228/* detect bugs with the vector support macros */ 229void validate_pffft_simd (void ); 230void validate_pffft_simd (void ) { 231float f [16 ]= {0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 }; 232v4sf_union a0 ,a1 ,a2 ,a3 ,t ,u ; 233memcpy (a0 .f ,f ,4 * sizeof (float )); 234memcpy (a1 .f ,f + 4 ,4 * sizeof (float )); 235memcpy (a2 .f ,f + 8 ,4 * sizeof (float )); 236memcpy (a3 .f ,f + 12 ,4 * sizeof (float )); 237 238t = a0 ;u = a1 ;t .v = VZERO (); 239printf ("VZERO=[%2g %2g %2g %2g]\n" ,t .f [0 ],t .f [1 ],t .f [2 ],t .f [3 ]);assertv4 (t ,0 ,0 ,0 ,0 ); 240t .v = VADD (a1 .v ,a2 .v ); 241printf ("VADD(4:7,8:11)=[%2g %2g %2g %2g]\n" ,t .f [0 ],t .f [1 ],t .f [2 ],t .f [3 ]);assertv4 (t ,12 ,14 ,16 ,18 ); 242t .v = VMUL (a1 .v ,a2 .v ); 243printf ("VMUL(4:7,8:11)=[%2g %2g %2g %2g]\n" ,t .f [0 ],t .f [1 ],t .f [2 ],t .f [3 ]);assertv4 (t ,32 ,45 ,60 ,77 ); 244t .v = VMADD (a1 .v ,a2 .v ,a0 .v ); 245printf ("VMADD(4:7,8:11,0:3)=[%2g %2g %2g %2g]\n" ,t .f [0 ],t .f [1 ],t .f [2 ],t .f [3 ]);assertv4 (t ,32 ,46 ,62 ,80 ); 246INTERLEAVE2 (a1 .v ,a2 .v ,t .v ,u .v ); 247printf ("INTERLEAVE2(4:7,8:11)=[%2g %2g %2g %2g] [%2g %2g %2g %2g]\n" ,t .f [0 ],t .f [1 ],t .f [2 ],t .f [3 ],u .f [0 ],u .f [1 ],u .f [2 ],u .f [3 ]); 248assertv4 (t ,4 ,8 ,5 ,9 );assertv4 (u ,6 ,10 ,7 ,11 ); 249UNINTERLEAVE2 (a1 .v ,a2 .v ,t .v ,u .v ); 250printf ("UNINTERLEAVE2(4:7,8:11)=[%2g %2g %2g %2g] [%2g %2g %2g %2g]\n" ,t .f [0 ],t .f [1 ],t .f [2 ],t .f [3 ],u .f [0 ],u .f [1 ],u .f [2 ],u .f [3 ]); 251assertv4 (t ,4 ,6 ,8 ,10 );assertv4 (u ,5 ,7 ,9 ,11 ); 252 253t .v = LD_PS1 (f [15 ]); 254printf ("LD_PS1(15)=[%2g %2g %2g %2g]\n" ,t .f [0 ],t .f [1 ],t .f [2 ],t .f [3 ]); 255assertv4 (t ,15 ,15 ,15 ,15 ); 256t .v = VSWAPHL (a1 .v ,a2 .v ); 257printf ("VSWAPHL(4:7,8:11)=[%2g %2g %2g %2g]\n" ,t .f [0 ],t .f [1 ],t .f [2 ],t .f [3 ]); 258assertv4 (t ,8 ,9 ,6 ,7 ); 259VTRANSPOSE4 (a0 .v ,a1 .v ,a2 .v ,a3 .v ); 260printf ("VTRANSPOSE4(0:3,4:7,8:11,12:15)=[%2g %2g %2g %2g] [%2g %2g %2g %2g] [%2g %2g %2g %2g] [%2g %2g %2g %2g]\n" , 261a0 .f [0 ],a0 .f [1 ],a0 .f [2 ],a0 .f [3 ],a1 .f [0 ],a1 .f [1 ],a1 .f [2 ],a1 .f [3 ], 262a2 .f [0 ],a2 .f [1 ],a2 .f [2 ],a2 .f [3 ],a3 .f [0 ],a3 .f [1 ],a3 .f [2 ],a3 .f [3 ]); 263assertv4 (a0 ,0 ,4 ,8 ,12 );assertv4 (a1 ,1 ,5 ,9 ,13 );assertv4 (a2 ,2 ,6 ,10 ,14 );assertv4 (a3 ,3 ,7 ,11 ,15 ); 264} 265#endif 266#endif /*!PFFFT_SIMD_DISABLE */ 267 268#if 0 269/* SSE and co like 16-bytes aligned pointers */ 270#define MALLOC_V4SF_ALIGNMENT 64/* with a 64-byte alignment, we are even aligned on L2 cache lines... */ 271void * pffft_aligned_malloc (size_t nb_bytes ) { 272void * p ,* p0 = malloc (nb_bytes + MALLOC_V4SF_ALIGNMENT ); 273if (!p0 )return (void * )0 ; 274p = (void * ) (((size_t )p0 + MALLOC_V4SF_ALIGNMENT )& (~((size_t ) (MALLOC_V4SF_ALIGNMENT - 1 )))); 275* ((void ** )p - 1 )= p0 ; 276return p ; 277} 278 279void pffft_aligned_free (void * p ) { 280if (p )free (* ((void ** )p - 1 )); 281} 282 283int pffft_simd_size () {return SIMD_SZ ; } 284#endif 285 286/* 287passf2 and passb2 has been merged here, fsign = -1 for passf2, +1 for passb2 288*/ 289static NEVER_INLINE (void )passf2_ps (int ido ,int l1 ,const v4sf * cc ,v4sf * ch ,const float * wa1 ,float fsign ) { 290int k ,i ; 291int l1ido = l1 * ido ; 292if (ido <=2 ) { 293for (k = 0 ;k < l1ido ;k += ido ,ch += ido ,cc += 2 * ido ) { 294ch [0 ]= VADD (cc [0 ],cc [ido + 0 ]); 295ch [l1ido ]= VSUB (cc [0 ],cc [ido + 0 ]); 296ch [1 ]= VADD (cc [1 ],cc [ido + 1 ]); 297ch [l1ido + 1 ]= VSUB (cc [1 ],cc [ido + 1 ]); 298 } 299 }else { 300for (k = 0 ;k < l1ido ;k += ido ,ch += ido ,cc += 2 * ido ) { 301for (i = 0 ;i < ido - 1 ;i += 2 ) { 302v4sf tr2 = VSUB (cc [i + 0 ],cc [i + ido + 0 ]); 303v4sf ti2 = VSUB (cc [i + 1 ],cc [i + ido + 1 ]); 304v4sf wr = LD_PS1 (wa1 [i ]),wi = VMUL (LD_PS1 (fsign ),LD_PS1 (wa1 [i + 1 ])); 305ch [i ]= VADD (cc [i + 0 ],cc [i + ido + 0 ]); 306ch [i + 1 ]= VADD (cc [i + 1 ],cc [i + ido + 1 ]); 307VCPLXMUL (tr2 ,ti2 ,wr ,wi ); 308ch [i + l1ido ]= tr2 ; 309ch [i + l1ido + 1 ]= ti2 ; 310 } 311 } 312 } 313} 314 315/* 316passf3 and passb3 has been merged here, fsign = -1 for passf3, +1 for passb3 317*/ 318#if 0 319static NEVER_INLINE (void )passf3_ps (int ido ,int l1 ,const v4sf * cc ,v4sf * ch , 320const float * wa1 ,const float * wa2 ,float fsign ) { 321static const float taur = -0.5f ; 322float taui = 0.866025403784439f * fsign ; 323int i ,k ; 324v4sf tr2 ,ti2 ,cr2 ,ci2 ,cr3 ,ci3 ,dr2 ,di2 ,dr3 ,di3 ; 325int l1ido = l1 * ido ; 326float wr1 ,wi1 ,wr2 ,wi2 ; 327assert (ido > 2 ); 328for (k = 0 ;k < l1ido ;k += ido ,cc += 3 * ido ,ch += ido ) { 329for (i = 0 ;i < ido - 1 ;i += 2 ) { 330tr2 = VADD (cc [i + ido ],cc [i + 2 * ido ]); 331cr2 = VADD (cc [i ],SVMUL (taur ,tr2 )); 332ch [i ]= VADD (cc [i ],tr2 ); 333ti2 = VADD (cc [i + ido + 1 ],cc [i + 2 * ido + 1 ]); 334ci2 = VADD (cc [i + 1 ],SVMUL (taur ,ti2 )); 335ch [i + 1 ]= VADD (cc [i + 1 ],ti2 ); 336cr3 = SVMUL (taui ,VSUB (cc [i + ido ],cc [i + 2 * ido ])); 337ci3 = SVMUL (taui ,VSUB (cc [i + ido + 1 ],cc [i + 2 * ido + 1 ])); 338dr2 = VSUB (cr2 ,ci3 ); 339dr3 = VADD (cr2 ,ci3 ); 340di2 = VADD (ci2 ,cr3 ); 341di3 = VSUB (ci2 ,cr3 ); 342wr1 = wa1 [i ],wi1 = fsign * wa1 [i + 1 ],wr2 = wa2 [i ],wi2 = fsign * wa2 [i + 1 ]; 343VCPLXMUL (dr2 ,di2 ,LD_PS1 (wr1 ),LD_PS1 (wi1 )); 344ch [i + l1ido ]= dr2 ; 345ch [i + l1ido + 1 ]= di2 ; 346VCPLXMUL (dr3 ,di3 ,LD_PS1 (wr2 ),LD_PS1 (wi2 )); 347ch [i + 2 * l1ido ]= dr3 ; 348ch [i + 2 * l1ido + 1 ]= di3 ; 349 } 350 } 351}/* passf3 */ 352#endif 353 354static NEVER_INLINE (void )passf4_ps (int ido ,int l1 ,const v4sf * cc ,v4sf * ch , 355const float * wa1 ,const float * wa2 ,const float * wa3 ,float fsign ) { 356/* isign == -1 for forward transform and +1 for backward transform */ 357 358int i ,k ; 359v4sf ci2 ,ci3 ,ci4 ,cr2 ,cr3 ,cr4 ,ti1 ,ti2 ,ti3 ,ti4 ,tr1 ,tr2 ,tr3 ,tr4 ; 360int l1ido = l1 * ido ; 361if (ido == 2 ) { 362for (k = 0 ;k < l1ido ;k += ido ,ch += ido ,cc += 4 * ido ) { 363tr1 = VSUB (cc [0 ],cc [2 * ido + 0 ]); 364tr2 = VADD (cc [0 ],cc [2 * ido + 0 ]); 365ti1 = VSUB (cc [1 ],cc [2 * ido + 1 ]); 366ti2 = VADD (cc [1 ],cc [2 * ido + 1 ]); 367ti4 = VMUL (VSUB (cc [1 * ido + 0 ],cc [3 * ido + 0 ]),LD_PS1 (fsign )); 368tr4 = VMUL (VSUB (cc [3 * ido + 1 ],cc [1 * ido + 1 ]),LD_PS1 (fsign )); 369tr3 = VADD (cc [ido + 0 ],cc [3 * ido + 0 ]); 370ti3 = VADD (cc [ido + 1 ],cc [3 * ido + 1 ]); 371 372ch [0 * l1ido + 0 ]= VADD (tr2 ,tr3 ); 373ch [0 * l1ido + 1 ]= VADD (ti2 ,ti3 ); 374ch [1 * l1ido + 0 ]= VADD (tr1 ,tr4 ); 375ch [1 * l1ido + 1 ]= VADD (ti1 ,ti4 ); 376ch [2 * l1ido + 0 ]= VSUB (tr2 ,tr3 ); 377ch [2 * l1ido + 1 ]= VSUB (ti2 ,ti3 ); 378ch [3 * l1ido + 0 ]= VSUB (tr1 ,tr4 ); 379ch [3 * l1ido + 1 ]= VSUB (ti1 ,ti4 ); 380 } 381 }else { 382for (k = 0 ;k < l1ido ;k += ido ,ch += ido ,cc += 4 * ido ) { 383for (i = 0 ;i < ido - 1 ;i += 2 ) { 384float wr1 ,wi1 ,wr2 ,wi2 ,wr3 ,wi3 ; 385tr1 = VSUB (cc [i + 0 ],cc [i + 2 * ido + 0 ]); 386tr2 = VADD (cc [i + 0 ],cc [i + 2 * ido + 0 ]); 387ti1 = VSUB (cc [i + 1 ],cc [i + 2 * ido + 1 ]); 388ti2 = VADD (cc [i + 1 ],cc [i + 2 * ido + 1 ]); 389tr4 = VMUL (VSUB (cc [i + 3 * ido + 1 ],cc [i + 1 * ido + 1 ]),LD_PS1 (fsign )); 390ti4 = VMUL (VSUB (cc [i + 1 * ido + 0 ],cc [i + 3 * ido + 0 ]),LD_PS1 (fsign )); 391tr3 = VADD (cc [i + ido + 0 ],cc [i + 3 * ido + 0 ]); 392ti3 = VADD (cc [i + ido + 1 ],cc [i + 3 * ido + 1 ]); 393 394ch [i ]= VADD (tr2 ,tr3 ); 395cr3 = VSUB (tr2 ,tr3 ); 396ch [i + 1 ]= VADD (ti2 ,ti3 ); 397ci3 = VSUB (ti2 ,ti3 ); 398 399cr2 = VADD (tr1 ,tr4 ); 400cr4 = VSUB (tr1 ,tr4 ); 401ci2 = VADD (ti1 ,ti4 ); 402ci4 = VSUB (ti1 ,ti4 ); 403wr1 = wa1 [i ],wi1 = fsign * wa1 [i + 1 ]; 404VCPLXMUL (cr2 ,ci2 ,LD_PS1 (wr1 ),LD_PS1 (wi1 )); 405wr2 = wa2 [i ],wi2 = fsign * wa2 [i + 1 ]; 406ch [i + l1ido ]= cr2 ; 407ch [i + l1ido + 1 ]= ci2 ; 408 409VCPLXMUL (cr3 ,ci3 ,LD_PS1 (wr2 ),LD_PS1 (wi2 )); 410wr3 = wa3 [i ],wi3 = fsign * wa3 [i + 1 ]; 411ch [i + 2 * l1ido ]= cr3 ; 412ch [i + 2 * l1ido + 1 ]= ci3 ; 413 414VCPLXMUL (cr4 ,ci4 ,LD_PS1 (wr3 ),LD_PS1 (wi3 )); 415ch [i + 3 * l1ido ]= cr4 ; 416ch [i + 3 * l1ido + 1 ]= ci4 ; 417 } 418 } 419 } 420}/* passf4 */ 421 422#if 0 423/* 424passf5 and passb5 has been merged here, fsign = -1 for passf5, +1 for passb5 425*/ 426static NEVER_INLINE (void )passf5_ps (int ido ,int l1 ,const v4sf * cc ,v4sf * ch , 427const float * wa1 ,const float * wa2 , 428const float * wa3 ,const float * wa4 ,float fsign ) { 429static const float tr11 = .309016994374947f ; 430const float ti11 = .951056516295154f * fsign ; 431static const float tr12 = -.809016994374947f ; 432const float ti12 = .587785252292473f * fsign ; 433 434/* Local variables */ 435int i ,k ; 436v4sf ci2 ,ci3 ,ci4 ,ci5 ,di3 ,di4 ,di5 ,di2 ,cr2 ,cr3 ,cr5 ,cr4 ,ti2 ,ti3 , 437ti4 ,ti5 ,dr3 ,dr4 ,dr5 ,dr2 ,tr2 ,tr3 ,tr4 ,tr5 ; 438 439float wr1 ,wi1 ,wr2 ,wi2 ,wr3 ,wi3 ,wr4 ,wi4 ; 440 441#define cc_ref (a_1 ,a_2 ) cc[(a_2-1)*ido + a_1 + 1] 442#define ch_ref (a_1 ,a_3 ) ch[(a_3-1)*l1*ido + a_1 + 1] 443 444assert (ido > 2 ); 445for (k = 0 ;k < l1 ;++ k ,cc += 5 * ido ,ch += ido ) { 446for (i = 0 ;i < ido - 1 ;i += 2 ) { 447ti5 = VSUB (cc_ref (i ,2 ),cc_ref (i ,5 )); 448ti2 = VADD (cc_ref (i ,2 ),cc_ref (i ,5 )); 449ti4 = VSUB (cc_ref (i ,3 ),cc_ref (i ,4 )); 450ti3 = VADD (cc_ref (i ,3 ),cc_ref (i ,4 )); 451tr5 = VSUB (cc_ref (i - 1 ,2 ),cc_ref (i - 1 ,5 )); 452tr2 = VADD (cc_ref (i - 1 ,2 ),cc_ref (i - 1 ,5 )); 453tr4 = VSUB (cc_ref (i - 1 ,3 ),cc_ref (i - 1 ,4 )); 454tr3 = VADD (cc_ref (i - 1 ,3 ),cc_ref (i - 1 ,4 )); 455ch_ref (i - 1 ,1 )= VADD (cc_ref (i - 1 ,1 ),VADD (tr2 ,tr3 )); 456ch_ref (i ,1 )= VADD (cc_ref (i ,1 ),VADD (ti2 ,ti3 )); 457cr2 = VADD (cc_ref (i - 1 ,1 ),VADD (SVMUL (tr11 ,tr2 ),SVMUL (tr12 ,tr3 ))); 458ci2 = VADD (cc_ref (i ,1 ),VADD (SVMUL (tr11 ,ti2 ),SVMUL (tr12 ,ti3 ))); 459cr3 = VADD (cc_ref (i - 1 ,1 ),VADD (SVMUL (tr12 ,tr2 ),SVMUL (tr11 ,tr3 ))); 460ci3 = VADD (cc_ref (i ,1 ),VADD (SVMUL (tr12 ,ti2 ),SVMUL (tr11 ,ti3 ))); 461cr5 = VADD (SVMUL (ti11 ,tr5 ),SVMUL (ti12 ,tr4 )); 462ci5 = VADD (SVMUL (ti11 ,ti5 ),SVMUL (ti12 ,ti4 )); 463cr4 = VSUB (SVMUL (ti12 ,tr5 ),SVMUL (ti11 ,tr4 )); 464ci4 = VSUB (SVMUL (ti12 ,ti5 ),SVMUL (ti11 ,ti4 )); 465dr3 = VSUB (cr3 ,ci4 ); 466dr4 = VADD (cr3 ,ci4 ); 467di3 = VADD (ci3 ,cr4 ); 468di4 = VSUB (ci3 ,cr4 ); 469dr5 = VADD (cr2 ,ci5 ); 470dr2 = VSUB (cr2 ,ci5 ); 471di5 = VSUB (ci2 ,cr5 ); 472di2 = VADD (ci2 ,cr5 ); 473wr1 = wa1 [i ],wi1 = fsign * wa1 [i + 1 ],wr2 = wa2 [i ],wi2 = fsign * wa2 [i + 1 ]; 474wr3 = wa3 [i ],wi3 = fsign * wa3 [i + 1 ],wr4 = wa4 [i ],wi4 = fsign * wa4 [i + 1 ]; 475VCPLXMUL (dr2 ,di2 ,LD_PS1 (wr1 ),LD_PS1 (wi1 )); 476ch_ref (i - 1 ,2 )= dr2 ; 477ch_ref (i ,2 )= di2 ; 478VCPLXMUL (dr3 ,di3 ,LD_PS1 (wr2 ),LD_PS1 (wi2 )); 479ch_ref (i - 1 ,3 )= dr3 ; 480ch_ref (i ,3 )= di3 ; 481VCPLXMUL (dr4 ,di4 ,LD_PS1 (wr3 ),LD_PS1 (wi3 )); 482ch_ref (i - 1 ,4 )= dr4 ; 483ch_ref (i ,4 )= di4 ; 484VCPLXMUL (dr5 ,di5 ,LD_PS1 (wr4 ),LD_PS1 (wi4 )); 485ch_ref (i - 1 ,5 )= dr5 ; 486ch_ref (i ,5 )= di5 ; 487 } 488 } 489#undef ch_ref 490#undef cc_ref 491} 492#endif 493 494static NEVER_INLINE (void )radf2_ps (int ido ,int l1 ,const v4sf * RESTRICT cc ,v4sf * RESTRICT ch ,const float * wa1 ) { 495static const float minus_one = -1.f ; 496int i ,k ,l1ido = l1 * ido ; 497for (k = 0 ;k < l1ido ;k += ido ) { 498v4sf a = cc [k ],b = cc [k + l1ido ]; 499ch [2 * k ]= VADD (a ,b ); 500ch [2 * (k + ido )- 1 ]= VSUB (a ,b ); 501 } 502if (ido < 2 )return ; 503if (ido != 2 ) { 504for (k = 0 ;k < l1ido ;k += ido ) { 505for (i = 2 ;i < ido ;i += 2 ) { 506v4sf tr2 = cc [i - 1 + k + l1ido ],ti2 = cc [i + k + l1ido ]; 507v4sf br = cc [i - 1 + k ],bi = cc [i + k ]; 508VCPLXMULCONJ (tr2 ,ti2 ,LD_PS1 (wa1 [i - 2 ]),LD_PS1 (wa1 [i - 1 ])); 509ch [i + 2 * k ]= VADD (bi ,ti2 ); 510ch [2 * (k + ido )- i ]= VSUB (ti2 ,bi ); 511ch [i - 1 + 2 * k ]= VADD (br ,tr2 ); 512ch [2 * (k + ido )- i - 1 ]= VSUB (br ,tr2 ); 513 } 514 } 515if (ido %2 == 1 )return ; 516 } 517for (k = 0 ;k < l1ido ;k += ido ) { 518ch [2 * k + ido ]= SVMUL (minus_one ,cc [ido - 1 + k + l1ido ]); 519ch [2 * k + ido - 1 ]= cc [k + ido - 1 ]; 520 } 521}/* radf2 */ 522 523 524static NEVER_INLINE (void )radb2_ps (int ido ,int l1 ,const v4sf * cc ,v4sf * ch ,const float * wa1 ) { 525static const float minus_two = -2 ; 526int i ,k ,l1ido = l1 * ido ; 527v4sf a ,b ,c ,d ,tr2 ,ti2 ; 528for (k = 0 ;k < l1ido ;k += ido ) { 529a = cc [2 * k ];b = cc [2 * (k + ido )- 1 ]; 530ch [k ]= VADD (a ,b ); 531ch [k + l1ido ]= VSUB (a ,b ); 532 } 533if (ido < 2 )return ; 534if (ido != 2 ) { 535for (k = 0 ;k < l1ido ;k += ido ) { 536for (i = 2 ;i < ido ;i += 2 ) { 537a = cc [i - 1 + 2 * k ];b = cc [2 * (k + ido )- i - 1 ]; 538c = cc [i + 0 + 2 * k ];d = cc [2 * (k + ido )- i + 0 ]; 539ch [i - 1 + k ]= VADD (a ,b ); 540tr2 = VSUB (a ,b ); 541ch [i + 0 + k ]= VSUB (c ,d ); 542ti2 = VADD (c ,d ); 543VCPLXMUL (tr2 ,ti2 ,LD_PS1 (wa1 [i - 2 ]),LD_PS1 (wa1 [i - 1 ])); 544ch [i - 1 + k + l1ido ]= tr2 ; 545ch [i + 0 + k + l1ido ]= ti2 ; 546 } 547 } 548if (ido %2 == 1 )return ; 549 } 550for (k = 0 ;k < l1ido ;k += ido ) { 551a = cc [2 * k + ido - 1 ];b = cc [2 * k + ido ]; 552ch [k + ido - 1 ]= VADD (a ,a ); 553ch [k + ido - 1 + l1ido ]= SVMUL (minus_two ,b ); 554 } 555}/* radb2 */ 556 557#if 0 558static void radf3_ps (int ido ,int l1 ,const v4sf * RESTRICT cc ,v4sf * RESTRICT ch , 559const float * wa1 ,const float * wa2 ) { 560static const float taur = -0.5f ; 561static const float taui = 0.866025403784439f ; 562int i ,k ,ic ; 563v4sf ci2 ,di2 ,di3 ,cr2 ,dr2 ,dr3 ,ti2 ,ti3 ,tr2 ,tr3 ,wr1 ,wi1 ,wr2 ,wi2 ; 564for (k = 0 ;k < l1 ;k ++ ) { 565cr2 = VADD (cc [(k + l1 )* ido ],cc [(k + 2 * l1 )* ido ]); 566ch [3 * k * ido ]= VADD (cc [k * ido ],cr2 ); 567ch [(3 * k + 2 )* ido ]= SVMUL (taui ,VSUB (cc [(k + l1 * 2 )* ido ],cc [(k + l1 )* ido ])); 568ch [ido - 1 + (3 * k + 1 )* ido ]= VADD (cc [k * ido ],SVMUL (taur ,cr2 )); 569 } 570if (ido == 1 )return ; 571for (k = 0 ;k < l1 ;k ++ ) { 572for (i = 2 ;i < ido ;i += 2 ) { 573ic = ido - i ; 574wr1 = LD_PS1 (wa1 [i - 2 ]);wi1 = LD_PS1 (wa1 [i - 1 ]); 575dr2 = cc [i - 1 + (k + l1 )* ido ];di2 = cc [i + (k + l1 )* ido ]; 576VCPLXMULCONJ (dr2 ,di2 ,wr1 ,wi1 ); 577 578wr2 = LD_PS1 (wa2 [i - 2 ]);wi2 = LD_PS1 (wa2 [i - 1 ]); 579dr3 = cc [i - 1 + (k + l1 * 2 )* ido ];di3 = cc [i + (k + l1 * 2 )* ido ]; 580VCPLXMULCONJ (dr3 ,di3 ,wr2 ,wi2 ); 581 582cr2 = VADD (dr2 ,dr3 ); 583ci2 = VADD (di2 ,di3 ); 584ch [i - 1 + 3 * k * ido ]= VADD (cc [i - 1 + k * ido ],cr2 ); 585ch [i + 3 * k * ido ]= VADD (cc [i + k * ido ],ci2 ); 586tr2 = VADD (cc [i - 1 + k * ido ],SVMUL (taur ,cr2 )); 587ti2 = VADD (cc [i + k * ido ],SVMUL (taur ,ci2 )); 588tr3 = SVMUL (taui ,VSUB (di2 ,di3 )); 589ti3 = SVMUL (taui ,VSUB (dr3 ,dr2 )); 590ch [i - 1 + (3 * k + 2 )* ido ]= VADD (tr2 ,tr3 ); 591ch [ic - 1 + (3 * k + 1 )* ido ]= VSUB (tr2 ,tr3 ); 592ch [i + (3 * k + 2 )* ido ]= VADD (ti2 ,ti3 ); 593ch [ic + (3 * k + 1 )* ido ]= VSUB (ti3 ,ti2 ); 594 } 595 } 596}/* radf3 */ 597 598 599static void radb3_ps (int ido ,int l1 ,const v4sf * RESTRICT cc ,v4sf * RESTRICT ch , 600const float * wa1 ,const float * wa2 ) 601{ 602static const float taur = -0.5f ; 603static const float taui = 0.866025403784439f ; 604static const float taui_2 = 0.866025403784439f * 2 ; 605int i ,k ,ic ; 606v4sf ci2 ,ci3 ,di2 ,di3 ,cr2 ,cr3 ,dr2 ,dr3 ,ti2 ,tr2 ; 607for (k = 0 ;k < l1 ;k ++ ) { 608tr2 = cc [ido - 1 + (3 * k + 1 )* ido ];tr2 = VADD (tr2 ,tr2 ); 609cr2 = VMADD (LD_PS1 (taur ),tr2 ,cc [3 * k * ido ]); 610ch [k * ido ]= VADD (cc [3 * k * ido ],tr2 ); 611ci3 = SVMUL (taui_2 ,cc [(3 * k + 2 )* ido ]); 612ch [(k + l1 )* ido ]= VSUB (cr2 ,ci3 ); 613ch [(k + 2 * l1 )* ido ]= VADD (cr2 ,ci3 ); 614 } 615if (ido == 1 )return ; 616for (k = 0 ;k < l1 ;k ++ ) { 617for (i = 2 ;i < ido ;i += 2 ) { 618ic = ido - i ; 619tr2 = VADD (cc [i - 1 + (3 * k + 2 )* ido ],cc [ic - 1 + (3 * k + 1 )* ido ]); 620cr2 = VMADD (LD_PS1 (taur ),tr2 ,cc [i - 1 + 3 * k * ido ]); 621ch [i - 1 + k * ido ]= VADD (cc [i - 1 + 3 * k * ido ],tr2 ); 622ti2 = VSUB (cc [i + (3 * k + 2 )* ido ],cc [ic + (3 * k + 1 )* ido ]); 623ci2 = VMADD (LD_PS1 (taur ),ti2 ,cc [i + 3 * k * ido ]); 624ch [i + k * ido ]= VADD (cc [i + 3 * k * ido ],ti2 ); 625cr3 = SVMUL (taui ,VSUB (cc [i - 1 + (3 * k + 2 )* ido ],cc [ic - 1 + (3 * k + 1 )* ido ])); 626ci3 = SVMUL (taui ,VADD (cc [i + (3 * k + 2 )* ido ],cc [ic + (3 * k + 1 )* ido ])); 627dr2 = VSUB (cr2 ,ci3 ); 628dr3 = VADD (cr2 ,ci3 ); 629di2 = VADD (ci2 ,cr3 ); 630di3 = VSUB (ci2 ,cr3 ); 631VCPLXMUL (dr2 ,di2 ,LD_PS1 (wa1 [i - 2 ]),LD_PS1 (wa1 [i - 1 ])); 632ch [i - 1 + (k + l1 )* ido ]= dr2 ; 633ch [i + (k + l1 )* ido ]= di2 ; 634VCPLXMUL (dr3 ,di3 ,LD_PS1 (wa2 [i - 2 ]),LD_PS1 (wa2 [i - 1 ])); 635ch [i - 1 + (k + 2 * l1 )* ido ]= dr3 ; 636ch [i + (k + 2 * l1 )* ido ]= di3 ; 637 } 638 } 639}/* radb3 */ 640#endif 641 642static NEVER_INLINE (void )radf4_ps (int ido ,int l1 ,const v4sf * RESTRICT cc ,v4sf * RESTRICT ch , 643const float * RESTRICT wa1 ,const float * RESTRICT wa2 ,const float * RESTRICT wa3 ) 644{ 645static const float minus_hsqt2 = (float )-0.7071067811865475 ; 646int i ,k ,l1ido = l1 * ido ; 647 { 648const v4sf * RESTRICT cc_ = cc ,* RESTRICT cc_end = cc + l1ido ; 649v4sf * RESTRICT ch_ = ch ; 650while (cc < cc_end ) { 651/* this loop represents between 25% and 40% of total radf4_ps cost ! */ 652v4sf a0 = cc [0 ],a1 = cc [l1ido ]; 653v4sf a2 = cc [2 * l1ido ],a3 = cc [3 * l1ido ]; 654v4sf tr1 = VADD (a1 ,a3 ); 655v4sf tr2 = VADD (a0 ,a2 ); 656ch [2 * ido - 1 ]= VSUB (a0 ,a2 ); 657ch [2 * ido ]= VSUB (a3 ,a1 ); 658ch [0 ]= VADD (tr1 ,tr2 ); 659ch [4 * ido - 1 ]= VSUB (tr2 ,tr1 ); 660cc += ido ;ch += 4 * ido ; 661 } 662cc = cc_ ;ch = ch_ ; 663 } 664if (ido < 2 )return ; 665if (ido != 2 ) { 666for (k = 0 ;k < l1ido ;k += ido ) { 667const v4sf * RESTRICT pc = (v4sf * )(cc + 1 + k ); 668for (i = 2 ;i < ido ;i += 2 ,pc += 2 ) { 669int ic = ido - i ; 670v4sf wr ,wi ,cr2 ,ci2 ,cr3 ,ci3 ,cr4 ,ci4 ; 671v4sf tr1 ,ti1 ,tr2 ,ti2 ,tr3 ,ti3 ,tr4 ,ti4 ; 672 673cr2 = pc [1 * l1ido + 0 ]; 674ci2 = pc [1 * l1ido + 1 ]; 675wr = LD_PS1 (wa1 [i - 2 ]); 676wi = LD_PS1 (wa1 [i - 1 ]); 677VCPLXMULCONJ (cr2 ,ci2 ,wr ,wi ); 678 679cr3 = pc [2 * l1ido + 0 ]; 680ci3 = pc [2 * l1ido + 1 ]; 681wr = LD_PS1 (wa2 [i - 2 ]); 682wi = LD_PS1 (wa2 [i - 1 ]); 683VCPLXMULCONJ (cr3 ,ci3 ,wr ,wi ); 684 685cr4 = pc [3 * l1ido ]; 686ci4 = pc [3 * l1ido + 1 ]; 687wr = LD_PS1 (wa3 [i - 2 ]); 688wi = LD_PS1 (wa3 [i - 1 ]); 689VCPLXMULCONJ (cr4 ,ci4 ,wr ,wi ); 690 691/* at this point, on SSE, five of "cr2 cr3 cr4 ci2 ci3 ci4" should be loaded in registers */ 692 693tr1 = VADD (cr2 ,cr4 ); 694tr4 = VSUB (cr4 ,cr2 ); 695tr2 = VADD (pc [0 ],cr3 ); 696tr3 = VSUB (pc [0 ],cr3 ); 697ch [i - 1 + 4 * k ]= VADD (tr1 ,tr2 ); 698ch [ic - 1 + 4 * k + 3 * ido ]= VSUB (tr2 ,tr1 );/* at this point tr1 and tr2 can be disposed */ 699ti1 = VADD (ci2 ,ci4 ); 700ti4 = VSUB (ci2 ,ci4 ); 701ch [i - 1 + 4 * k + 2 * ido ]= VADD (ti4 ,tr3 ); 702ch [ic - 1 + 4 * k + 1 * ido ]= VSUB (tr3 ,ti4 );/* dispose tr3, ti4 */ 703ti2 = VADD (pc [1 ],ci3 ); 704ti3 = VSUB (pc [1 ],ci3 ); 705ch [i + 4 * k ]= VADD (ti1 ,ti2 ); 706ch [ic + 4 * k + 3 * ido ]= VSUB (ti1 ,ti2 ); 707ch [i + 4 * k + 2 * ido ]= VADD (tr4 ,ti3 ); 708ch [ic + 4 * k + 1 * ido ]= VSUB (tr4 ,ti3 ); 709 } 710 } 711if (ido %2 == 1 )return ; 712 } 713for (k = 0 ;k < l1ido ;k += ido ) { 714v4sf a = cc [ido - 1 + k + l1ido ],b = cc [ido - 1 + k + 3 * l1ido ]; 715v4sf c = cc [ido - 1 + k ],d = cc [ido - 1 + k + 2 * l1ido ]; 716v4sf ti1 = SVMUL (minus_hsqt2 ,VADD (a ,b )); 717v4sf tr1 = SVMUL (minus_hsqt2 ,VSUB (b ,a )); 718ch [ido - 1 + 4 * k ]= VADD (tr1 ,c ); 719ch [ido - 1 + 4 * k + 2 * ido ]= VSUB (c ,tr1 ); 720ch [4 * k + 1 * ido ]= VSUB (ti1 ,d ); 721ch [4 * k + 3 * ido ]= VADD (ti1 ,d ); 722 } 723}/* radf4 */ 724 725 726static NEVER_INLINE (void )radb4_ps (int ido ,int l1 ,const v4sf * RESTRICT cc ,v4sf * RESTRICT ch , 727const float * RESTRICT wa1 ,const float * RESTRICT wa2 ,const float * RESTRICT wa3 ) 728{ 729static const float minus_sqrt2 = (float )-1.414213562373095 ; 730static const float two = 2.f ; 731int i ,k ,l1ido = l1 * ido ; 732v4sf ci2 ,ci3 ,ci4 ,cr2 ,cr3 ,cr4 ,ti1 ,ti2 ,ti3 ,ti4 ,tr1 ,tr2 ,tr3 ,tr4 ; 733 { 734const v4sf * RESTRICT cc_ = cc ,* RESTRICT ch_end = ch + l1ido ; 735v4sf * ch_ = ch ; 736while (ch < ch_end ) { 737v4sf a = cc [0 ],b = cc [4 * ido - 1 ]; 738v4sf c = cc [2 * ido ],d = cc [2 * ido - 1 ]; 739tr3 = SVMUL (two ,d ); 740tr2 = VADD (a ,b ); 741tr1 = VSUB (a ,b ); 742tr4 = SVMUL (two ,c ); 743ch [0 * l1ido ]= VADD (tr2 ,tr3 ); 744ch [2 * l1ido ]= VSUB (tr2 ,tr3 ); 745ch [1 * l1ido ]= VSUB (tr1 ,tr4 ); 746ch [3 * l1ido ]= VADD (tr1 ,tr4 ); 747 748cc += 4 * ido ;ch += ido ; 749 } 750cc = cc_ ;ch = ch_ ; 751 } 752if (ido < 2 )return ; 753if (ido != 2 ) { 754for (k = 0 ;k < l1ido ;k += ido ) { 755const v4sf * RESTRICT pc = (v4sf * )(cc - 1 + 4 * k ); 756v4sf * RESTRICT ph = (v4sf * )(ch + k + 1 ); 757for (i = 2 ;i < ido ;i += 2 ) { 758 759tr1 = VSUB (pc [i ],pc [4 * ido - i ]); 760tr2 = VADD (pc [i ],pc [4 * ido - i ]); 761ti4 = VSUB (pc [2 * ido + i ],pc [2 * ido - i ]); 762tr3 = VADD (pc [2 * ido + i ],pc [2 * ido - i ]); 763ph [0 ]= VADD (tr2 ,tr3 ); 764cr3 = VSUB (tr2 ,tr3 ); 765 766ti3 = VSUB (pc [2 * ido + i + 1 ],pc [2 * ido - i + 1 ]); 767tr4 = VADD (pc [2 * ido + i + 1 ],pc [2 * ido - i + 1 ]); 768cr2 = VSUB (tr1 ,tr4 ); 769cr4 = VADD (tr1 ,tr4 ); 770 771ti1 = VADD (pc [i + 1 ],pc [4 * ido - i + 1 ]); 772ti2 = VSUB (pc [i + 1 ],pc [4 * ido - i + 1 ]); 773 774ph [1 ]= VADD (ti2 ,ti3 );ph += l1ido ; 775ci3 = VSUB (ti2 ,ti3 ); 776ci2 = VADD (ti1 ,ti4 ); 777ci4 = VSUB (ti1 ,ti4 ); 778VCPLXMUL (cr2 ,ci2 ,LD_PS1 (wa1 [i - 2 ]),LD_PS1 (wa1 [i - 1 ])); 779ph [0 ]= cr2 ; 780ph [1 ]= ci2 ;ph += l1ido ; 781VCPLXMUL (cr3 ,ci3 ,LD_PS1 (wa2 [i - 2 ]),LD_PS1 (wa2 [i - 1 ])); 782ph [0 ]= cr3 ; 783ph [1 ]= ci3 ;ph += l1ido ; 784VCPLXMUL (cr4 ,ci4 ,LD_PS1 (wa3 [i - 2 ]),LD_PS1 (wa3 [i - 1 ])); 785ph [0 ]= cr4 ; 786ph [1 ]= ci4 ;ph = ph - 3 * l1ido + 2 ; 787 } 788 } 789if (ido %2 == 1 )return ; 790 } 791for (k = 0 ;k < l1ido ;k += ido ) { 792int i0 = 4 * k + ido ; 793v4sf c = cc [i0 - 1 ],d = cc [i0 + 2 * ido - 1 ]; 794v4sf a = cc [i0 + 0 ],b = cc [i0 + 2 * ido + 0 ]; 795tr1 = VSUB (c ,d ); 796tr2 = VADD (c ,d ); 797ti1 = VADD (b ,a ); 798ti2 = VSUB (b ,a ); 799ch [ido - 1 + k + 0 * l1ido ]= VADD (tr2 ,tr2 ); 800ch [ido - 1 + k + 1 * l1ido ]= SVMUL (minus_sqrt2 ,VSUB (ti1 ,tr1 )); 801ch [ido - 1 + k + 2 * l1ido ]= VADD (ti2 ,ti2 ); 802ch [ido - 1 + k + 3 * l1ido ]= SVMUL (minus_sqrt2 ,VADD (ti1 ,tr1 )); 803 } 804}/* radb4 */ 805 806#if 0 807static void radf5_ps (int ido ,int l1 ,const v4sf * RESTRICT cc ,v4sf * RESTRICT ch , 808const float * wa1 ,const float * wa2 ,const float * wa3 ,const float * wa4 ) 809{ 810static const float tr11 = .309016994374947f ; 811static const float ti11 = .951056516295154f ; 812static const float tr12 = -.809016994374947f ; 813static const float ti12 = .587785252292473f ; 814 815/* System generated locals */ 816int cc_offset ,ch_offset ; 817 818/* Local variables */ 819int i ,k ,ic ; 820v4sf ci2 ,di2 ,ci4 ,ci5 ,di3 ,di4 ,di5 ,ci3 ,cr2 ,cr3 ,dr2 ,dr3 ,dr4 ,dr5 , 821cr5 ,cr4 ,ti2 ,ti3 ,ti5 ,ti4 ,tr2 ,tr3 ,tr4 ,tr5 ; 822int idp2 ; 823 824 825#define cc_ref (a_1 ,a_2 ,a_3 ) cc[((a_3)*l1 + (a_2))*ido + a_1] 826#define ch_ref (a_1 ,a_2 ,a_3 ) ch[((a_3)*5 + (a_2))*ido + a_1] 827 828/* Parameter adjustments */ 829ch_offset = 1 + ido * 6 ; 830ch -= ch_offset ; 831cc_offset = 1 + ido * (1 + l1 ); 832cc -= cc_offset ; 833 834/* Function Body */ 835for (k = 1 ;k <=l1 ;++ k ) { 836cr2 = VADD (cc_ref (1 ,k ,5 ),cc_ref (1 ,k ,2 )); 837ci5 = VSUB (cc_ref (1 ,k ,5 ),cc_ref (1 ,k ,2 )); 838cr3 = VADD (cc_ref (1 ,k ,4 ),cc_ref (1 ,k ,3 )); 839ci4 = VSUB (cc_ref (1 ,k ,4 ),cc_ref (1 ,k ,3 )); 840ch_ref (1 ,1 ,k )= VADD (cc_ref (1 ,k ,1 ),VADD (cr2 ,cr3 )); 841ch_ref (ido ,2 ,k )= VADD (cc_ref (1 ,k ,1 ),VADD (SVMUL (tr11 ,cr2 ),SVMUL (tr12 ,cr3 ))); 842ch_ref (1 ,3 ,k )= VADD (SVMUL (ti11 ,ci5 ),SVMUL (ti12 ,ci4 )); 843ch_ref (ido ,4 ,k )= VADD (cc_ref (1 ,k ,1 ),VADD (SVMUL (tr12 ,cr2 ),SVMUL (tr11 ,cr3 ))); 844ch_ref (1 ,5 ,k )= VSUB (SVMUL (ti12 ,ci5 ),SVMUL (ti11 ,ci4 )); 845/*printf("pffft: radf5, k=%d ch_ref=%f, ci4=%f\n", k, ch_ref(1, 5, k), ci4); */ 846 } 847if (ido == 1 ) { 848return ; 849 } 850idp2 = ido + 2 ; 851for (k = 1 ;k <=l1 ;++ k ) { 852for (i = 3 ;i <=ido ;i += 2 ) { 853ic = idp2 - i ; 854dr2 = LD_PS1 (wa1 [i - 3 ]);di2 = LD_PS1 (wa1 [i - 2 ]); 855dr3 = LD_PS1 (wa2 [i - 3 ]);di3 = LD_PS1 (wa2 [i - 2 ]); 856dr4 = LD_PS1 (wa3 [i - 3 ]);di4 = LD_PS1 (wa3 [i - 2 ]); 857dr5 = LD_PS1 (wa4 [i - 3 ]);di5 = LD_PS1 (wa4 [i - 2 ]); 858VCPLXMULCONJ (dr2 ,di2 ,cc_ref (i - 1 ,k ,2 ),cc_ref (i ,k ,2 )); 859VCPLXMULCONJ (dr3 ,di3 ,cc_ref (i - 1 ,k ,3 ),cc_ref (i ,k ,3 )); 860VCPLXMULCONJ (dr4 ,di4 ,cc_ref (i - 1 ,k ,4 ),cc_ref (i ,k ,4 )); 861VCPLXMULCONJ (dr5 ,di5 ,cc_ref (i - 1 ,k ,5 ),cc_ref (i ,k ,5 )); 862cr2 = VADD (dr2 ,dr5 ); 863ci5 = VSUB (dr5 ,dr2 ); 864cr5 = VSUB (di2 ,di5 ); 865ci2 = VADD (di2 ,di5 ); 866cr3 = VADD (dr3 ,dr4 ); 867ci4 = VSUB (dr4 ,dr3 ); 868cr4 = VSUB (di3 ,di4 ); 869ci3 = VADD (di3 ,di4 ); 870ch_ref (i - 1 ,1 ,k )= VADD (cc_ref (i - 1 ,k ,1 ),VADD (cr2 ,cr3 )); 871ch_ref (i ,1 ,k )= VSUB (cc_ref (i ,k ,1 ),VADD (ci2 ,ci3 ));/* */ 872tr2 = VADD (cc_ref (i - 1 ,k ,1 ),VADD (SVMUL (tr11 ,cr2 ),SVMUL (tr12 ,cr3 ))); 873ti2 = VSUB (cc_ref (i ,k ,1 ),VADD (SVMUL (tr11 ,ci2 ),SVMUL (tr12 ,ci3 )));/* */ 874tr3 = VADD (cc_ref (i - 1 ,k ,1 ),VADD (SVMUL (tr12 ,cr2 ),SVMUL (tr11 ,cr3 ))); 875ti3 = VSUB (cc_ref (i ,k ,1 ),VADD (SVMUL (tr12 ,ci2 ),SVMUL (tr11 ,ci3 )));/* */ 876tr5 = VADD (SVMUL (ti11 ,cr5 ),SVMUL (ti12 ,cr4 )); 877ti5 = VADD (SVMUL (ti11 ,ci5 ),SVMUL (ti12 ,ci4 )); 878tr4 = VSUB (SVMUL (ti12 ,cr5 ),SVMUL (ti11 ,cr4 )); 879ti4 = VSUB (SVMUL (ti12 ,ci5 ),SVMUL (ti11 ,ci4 )); 880ch_ref (i - 1 ,3 ,k )= VSUB (tr2 ,tr5 ); 881ch_ref (ic - 1 ,2 ,k )= VADD (tr2 ,tr5 ); 882ch_ref (i ,3 ,k )= VADD (ti2 ,ti5 ); 883ch_ref (ic ,2 ,k )= VSUB (ti5 ,ti2 ); 884ch_ref (i - 1 ,5 ,k )= VSUB (tr3 ,tr4 ); 885ch_ref (ic - 1 ,4 ,k )= VADD (tr3 ,tr4 ); 886ch_ref (i ,5 ,k )= VADD (ti3 ,ti4 ); 887ch_ref (ic ,4 ,k )= VSUB (ti4 ,ti3 ); 888 } 889 } 890#undef cc_ref 891#undef ch_ref 892}/* radf5 */ 893 894static void radb5_ps (int ido ,int l1 ,const v4sf * RESTRICT cc ,v4sf * RESTRICT ch , 895const float * wa1 ,const float * wa2 ,const float * wa3 ,const float * wa4 ) 896{ 897static const float tr11 = .309016994374947f ; 898static const float ti11 = .951056516295154f ; 899static const float tr12 = -.809016994374947f ; 900static const float ti12 = .587785252292473f ; 901 902int cc_offset ,ch_offset ; 903 904/* Local variables */ 905int i ,k ,ic ; 906v4sf ci2 ,ci3 ,ci4 ,ci5 ,di3 ,di4 ,di5 ,di2 ,cr2 ,cr3 ,cr5 ,cr4 ,ti2 ,ti3 , 907ti4 ,ti5 ,dr3 ,dr4 ,dr5 ,dr2 ,tr2 ,tr3 ,tr4 ,tr5 ; 908int idp2 ; 909 910#define cc_ref (a_1 ,a_2 ,a_3 ) cc[((a_3)*5 + (a_2))*ido + a_1] 911#define ch_ref (a_1 ,a_2 ,a_3 ) ch[((a_3)*l1 + (a_2))*ido + a_1] 912 913/* Parameter adjustments */ 914ch_offset = 1 + ido * (1 + l1 ); 915ch -= ch_offset ; 916cc_offset = 1 + ido * 6 ; 917cc -= cc_offset ; 918 919/* Function Body */ 920for (k = 1 ;k <=l1 ;++ k ) { 921ti5 = VADD (cc_ref (1 ,3 ,k ),cc_ref (1 ,3 ,k )); 922ti4 = VADD (cc_ref (1 ,5 ,k ),cc_ref (1 ,5 ,k )); 923tr2 = VADD (cc_ref (ido ,2 ,k ),cc_ref (ido ,2 ,k )); 924tr3 = VADD (cc_ref (ido ,4 ,k ),cc_ref (ido ,4 ,k )); 925ch_ref (1 ,k ,1 )= VADD (cc_ref (1 ,1 ,k ),VADD (tr2 ,tr3 )); 926cr2 = VADD (cc_ref (1 ,1 ,k ),VADD (SVMUL (tr11 ,tr2 ),SVMUL (tr12 ,tr3 ))); 927cr3 = VADD (cc_ref (1 ,1 ,k ),VADD (SVMUL (tr12 ,tr2 ),SVMUL (tr11 ,tr3 ))); 928ci5 = VADD (SVMUL (ti11 ,ti5 ),SVMUL (ti12 ,ti4 )); 929ci4 = VSUB (SVMUL (ti12 ,ti5 ),SVMUL (ti11 ,ti4 )); 930ch_ref (1 ,k ,2 )= VSUB (cr2 ,ci5 ); 931ch_ref (1 ,k ,3 )= VSUB (cr3 ,ci4 ); 932ch_ref (1 ,k ,4 )= VADD (cr3 ,ci4 ); 933ch_ref (1 ,k ,5 )= VADD (cr2 ,ci5 ); 934 } 935if (ido == 1 ) { 936return ; 937 } 938idp2 = ido + 2 ; 939for (k = 1 ;k <=l1 ;++ k ) { 940for (i = 3 ;i <=ido ;i += 2 ) { 941ic = idp2 - i ; 942ti5 = VADD (cc_ref (i ,3 ,k ),cc_ref (ic ,2 ,k )); 943ti2 = VSUB (cc_ref (i ,3 ,k ),cc_ref (ic ,2 ,k )); 944ti4 = VADD (cc_ref (i ,5 ,k ),cc_ref (ic ,4 ,k )); 945ti3 = VSUB (cc_ref (i ,5 ,k ),cc_ref (ic ,4 ,k )); 946tr5 = VSUB (cc_ref (i - 1 ,3 ,k ),cc_ref (ic - 1 ,2 ,k )); 947tr2 = VADD (cc_ref (i - 1 ,3 ,k ),cc_ref (ic - 1 ,2 ,k )); 948tr4 = VSUB (cc_ref (i - 1 ,5 ,k ),cc_ref (ic - 1 ,4 ,k )); 949tr3 = VADD (cc_ref (i - 1 ,5 ,k ),cc_ref (ic - 1 ,4 ,k )); 950ch_ref (i - 1 ,k ,1 )= VADD (cc_ref (i - 1 ,1 ,k ),VADD (tr2 ,tr3 )); 951ch_ref (i ,k ,1 )= VADD (cc_ref (i ,1 ,k ),VADD (ti2 ,ti3 )); 952cr2 = VADD (cc_ref (i - 1 ,1 ,k ),VADD (SVMUL (tr11 ,tr2 ),SVMUL (tr12 ,tr3 ))); 953ci2 = VADD (cc_ref (i ,1 ,k ),VADD (SVMUL (tr11 ,ti2 ),SVMUL (tr12 ,ti3 ))); 954cr3 = VADD (cc_ref (i - 1 ,1 ,k ),VADD (SVMUL (tr12 ,tr2 ),SVMUL (tr11 ,tr3 ))); 955ci3 = VADD (cc_ref (i ,1 ,k ),VADD (SVMUL (tr12 ,ti2 ),SVMUL (tr11 ,ti3 ))); 956cr5 = VADD (SVMUL (ti11 ,tr5 ),SVMUL (ti12 ,tr4 )); 957ci5 = VADD (SVMUL (ti11 ,ti5 ),SVMUL (ti12 ,ti4 )); 958cr4 = VSUB (SVMUL (ti12 ,tr5 ),SVMUL (ti11 ,tr4 )); 959ci4 = VSUB (SVMUL (ti12 ,ti5 ),SVMUL (ti11 ,ti4 )); 960dr3 = VSUB (cr3 ,ci4 ); 961dr4 = VADD (cr3 ,ci4 ); 962di3 = VADD (ci3 ,cr4 ); 963di4 = VSUB (ci3 ,cr4 ); 964dr5 = VADD (cr2 ,ci5 ); 965dr2 = VSUB (cr2 ,ci5 ); 966di5 = VSUB (ci2 ,cr5 ); 967di2 = VADD (ci2 ,cr5 ); 968VCPLXMUL (dr2 ,di2 ,LD_PS1 (wa1 [i - 3 ]),LD_PS1 (wa1 [i - 2 ])); 969VCPLXMUL (dr3 ,di3 ,LD_PS1 (wa2 [i - 3 ]),LD_PS1 (wa2 [i - 2 ])); 970VCPLXMUL (dr4 ,di4 ,LD_PS1 (wa3 [i - 3 ]),LD_PS1 (wa3 [i - 2 ])); 971VCPLXMUL (dr5 ,di5 ,LD_PS1 (wa4 [i - 3 ]),LD_PS1 (wa4 [i - 2 ])); 972 973ch_ref (i - 1 ,k ,2 )= dr2 ;ch_ref (i ,k ,2 )= di2 ; 974ch_ref (i - 1 ,k ,3 )= dr3 ;ch_ref (i ,k ,3 )= di3 ; 975ch_ref (i - 1 ,k ,4 )= dr4 ;ch_ref (i ,k ,4 )= di4 ; 976ch_ref (i - 1 ,k ,5 )= dr5 ;ch_ref (i ,k ,5 )= di5 ; 977 } 978 } 979#undef cc_ref 980#undef ch_ref 981}/* radb5 */ 982#endif 983 984static NEVER_INLINE (v4sf * )rfftf1_ps (int n ,const v4sf * input_readonly ,v4sf * work1 ,v4sf * work2 , 985const float * wa ,const int * ifac ) { 986v4sf * in = (v4sf * )input_readonly ; 987v4sf * out = (in == work2 ?work1 :work2 ); 988int nf = ifac [1 ],k1 ; 989int l2 = n ; 990int iw = n - 1 ; 991assert (in != out && work1 != work2 ); 992for (k1 = 1 ;k1 <=nf ;++ k1 ) { 993int kh = nf - k1 ; 994int ip = ifac [kh + 2 ]; 995int l1 = l2 /ip ; 996int ido = n /l2 ; 997iw -= (ip - 1 )* ido ; 998switch (ip ) { 999#if 0 1000case 5 : { 1001int ix2 = iw + ido ; 1002int ix3 = ix2 + ido ; 1003int ix4 = ix3 + ido ; 1004radf5_ps (ido ,l1 ,in ,out ,& wa [iw ],& wa [ix2 ],& wa [ix3 ],& wa [ix4 ]); 1005 }break ; 1006#endif 1007case 4 : { 1008int ix2 = iw + ido ; 1009int ix3 = ix2 + ido ; 1010radf4_ps (ido ,l1 ,in ,out ,& wa [iw ],& wa [ix2 ],& wa [ix3 ]); 1011 }break ; 1012#if 0 1013case 3 : { 1014int ix2 = iw + ido ; 1015radf3_ps (ido ,l1 ,in ,out ,& wa [iw ],& wa [ix2 ]); 1016 }break ; 1017#endif 1018case 2 : 1019radf2_ps (ido ,l1 ,in ,out ,& wa [iw ]); 1020break ; 1021default : 1022assert (0 ); 1023break ; 1024 } 1025l2 = l1 ; 1026if (out == work2 ) { 1027out = work1 ;in = work2 ; 1028 }else { 1029out = work2 ;in = work1 ; 1030 } 1031 } 1032return in ;/* this is in fact the output .. */ 1033}/* rfftf1 */ 1034 1035static NEVER_INLINE (v4sf * )rfftb1_ps (int n ,const v4sf * input_readonly ,v4sf * work1 ,v4sf * work2 , 1036const float * wa ,const int * ifac ) { 1037v4sf * in = (v4sf * )input_readonly ; 1038v4sf * out = (in == work2 ?work1 :work2 ); 1039int nf = ifac [1 ],k1 ; 1040int l1 = 1 ; 1041int iw = 0 ; 1042assert (in != out ); 1043for (k1 = 1 ;k1 <=nf ;k1 ++ ) { 1044int ip = ifac [k1 + 1 ]; 1045int l2 = ip * l1 ; 1046int ido = n /l2 ; 1047switch (ip ) { 1048#if 0 1049case 5 : { 1050int ix2 = iw + ido ; 1051int ix3 = ix2 + ido ; 1052int ix4 = ix3 + ido ; 1053radb5_ps (ido ,l1 ,in ,out ,& wa [iw ],& wa [ix2 ],& wa [ix3 ],& wa [ix4 ]); 1054 }break ; 1055#endif 1056case 4 : { 1057int ix2 = iw + ido ; 1058int ix3 = ix2 + ido ; 1059radb4_ps (ido ,l1 ,in ,out ,& wa [iw ],& wa [ix2 ],& wa [ix3 ]); 1060 }break ; 1061#if 0 1062case 3 : { 1063int ix2 = iw + ido ; 1064radb3_ps (ido ,l1 ,in ,out ,& wa [iw ],& wa [ix2 ]); 1065 }break ; 1066#endif 1067case 2 : 1068radb2_ps (ido ,l1 ,in ,out ,& wa [iw ]); 1069break ; 1070default : 1071assert (0 ); 1072break ; 1073 } 1074l1 = l2 ; 1075iw += (ip - 1 )* ido ; 1076 1077if (out == work2 ) { 1078out = work1 ;in = work2 ; 1079 }else { 1080out = work2 ;in = work1 ; 1081 } 1082 } 1083return in ;/* this is in fact the output .. */ 1084} 1085 1086static int decompose (int n ,int * ifac ,const int * ntryh ) { 1087int nl = n ,nf = 0 ,i ,j = 0 ; 1088for (j = 0 ;ntryh [j ];++ j ) { 1089int ntry = ntryh [j ]; 1090while (nl != 1 ) { 1091int nq = nl /ntry ; 1092int nr = nl - ntry * nq ; 1093if (nr == 0 ) { 1094ifac [2 + nf ++ ]= ntry ; 1095nl = nq ; 1096if (ntry == 2 && nf != 1 ) { 1097for (i = 2 ;i <=nf ;++ i ) { 1098int ib = nf - i + 2 ; 1099ifac [ib + 1 ]= ifac [ib ]; 1100 } 1101ifac [2 ]= 2 ; 1102 } 1103 }else break ; 1104 } 1105 } 1106ifac [0 ]= n ; 1107ifac [1 ]= nf ; 1108return nf ; 1109} 1110 1111 1112 1113static void rffti1_ps (int n ,float * wa ,int * ifac ) 1114{ 1115static const int ntryh []= {4 ,2 ,3 ,5 ,0 }; 1116int k1 ,j ,ii ; 1117 1118int nf = decompose (n ,ifac ,ntryh ); 1119float argh = (float )((2 * M_PI ) /n ); 1120int is = 0 ; 1121int nfm1 = nf - 1 ; 1122int l1 = 1 ; 1123for (k1 = 1 ;k1 <=nfm1 ;k1 ++ ) { 1124int ip = ifac [k1 + 1 ]; 1125int ld = 0 ; 1126int l2 = l1 * ip ; 1127int ido = n /l2 ; 1128int ipm = ip - 1 ; 1129for (j = 1 ;j <=ipm ;++ j ) { 1130float argld ; 1131int i = is ,fi = 0 ; 1132ld += l1 ; 1133argld = (float )ld * argh ; 1134for (ii = 3 ;ii <=ido ;ii += 2 ) { 1135i += 2 ; 1136fi += 1 ; 1137wa [i - 2 ]= cos ((float )fi * argld ); 1138wa [i - 1 ]= sin ((float )fi * argld ); 1139 } 1140is += ido ; 1141 } 1142l1 = l2 ; 1143 } 1144}/* rffti1 */ 1145 1146static 1147void cffti1_ps (int n ,float * wa ,int * ifac ) 1148{ 1149static const int ntryh []= {5 ,3 ,4 ,2 ,0 }; 1150int k1 ,j ,ii ; 1151 1152int nf = decompose (n ,ifac ,ntryh ); 1153float argh = (float )((2 * M_PI )/n ); 1154int i = 1 ; 1155int l1 = 1 ; 1156for (k1 = 1 ;k1 <=nf ;k1 ++ ) { 1157int ip = ifac [k1 + 1 ]; 1158int ld = 0 ; 1159int l2 = l1 * ip ; 1160int ido = n /l2 ; 1161int idot = ido + ido + 2 ; 1162int ipm = ip - 1 ; 1163for (j = 1 ;j <=ipm ;j ++ ) { 1164float argld ; 1165int i1 = i ,fi = 0 ; 1166wa [i - 1 ]= 1 ; 1167wa [i ]= 0 ; 1168ld += l1 ; 1169argld = (float )ld * argh ; 1170for (ii = 4 ;ii <=idot ;ii += 2 ) { 1171i += 2 ; 1172fi += 1 ; 1173wa [i - 1 ]= cos ((float )fi * argld ); 1174wa [i ]= sin ((float )fi * argld ); 1175 } 1176if (ip > 5 ) { 1177wa [i1 - 1 ]= wa [i - 1 ]; 1178wa [i1 ]= wa [i ]; 1179 } 1180 } 1181l1 = l2 ; 1182 } 1183}/* cffti1 */ 1184 1185 1186static 1187v4sf * cfftf1_ps (int n ,const v4sf * input_readonly ,v4sf * work1 ,v4sf * work2 ,const float * wa ,const int * ifac ,int isign ) { 1188v4sf * in = (v4sf * )input_readonly ; 1189v4sf * out = (in == work2 ?work1 :work2 ); 1190int nf = ifac [1 ],k1 ; 1191int l1 = 1 ; 1192int iw = 0 ; 1193assert (in != out && work1 != work2 ); 1194for (k1 = 2 ;k1 <=nf + 1 ;k1 ++ ) { 1195int ip = ifac [k1 ]; 1196int l2 = ip * l1 ; 1197int ido = n /l2 ; 1198int idot = ido + ido ; 1199switch (ip ) { 1200#if 0 1201case 5 : { 1202int ix2 = iw + idot ; 1203int ix3 = ix2 + idot ; 1204int ix4 = ix3 + idot ; 1205passf5_ps (idot ,l1 ,in ,out ,& wa [iw ],& wa [ix2 ],& wa [ix3 ],& wa [ix4 ], (float )isign ); 1206 }break ; 1207#endif 1208case 4 : { 1209int ix2 = iw + idot ; 1210int ix3 = ix2 + idot ; 1211passf4_ps (idot ,l1 ,in ,out ,& wa [iw ],& wa [ix2 ],& wa [ix3 ], (float )isign ); 1212 }break ; 1213case 2 : { 1214passf2_ps (idot ,l1 ,in ,out ,& wa [iw ], (float )isign ); 1215 }break ; 1216#if 0 1217case 3 : { 1218int ix2 = iw + idot ; 1219passf3_ps (idot ,l1 ,in ,out ,& wa [iw ],& wa [ix2 ], (float )isign ); 1220 }break ; 1221#endif 1222default : 1223assert (0 ); 1224 } 1225l1 = l2 ; 1226iw += (ip - 1 )* idot ; 1227if (out == work2 ) { 1228out = work1 ;in = work2 ; 1229 }else { 1230out = work2 ;in = work1 ; 1231 } 1232 } 1233 1234return in ;/* this is in fact the output .. */ 1235} 1236 1237 1238struct PFFFT_Setup { 1239int N ; 1240int Ncvec ;/* nb of complex simd vectors (N/4 if PFFFT_COMPLEX, N/8 if PFFFT_REAL) */ 1241int ifac [15 ]; 1242pffft_transform_t transform ; 1243v4sf * data ;/* allocated room for twiddle coefs */ 1244float * e ;/* points into 'data' , N/4*3 elements */ 1245float * twiddle ;/* points into 'data', N/4 elements */ 1246}; 1247 1248static 1249PFFFT_Setup * pffft_new_setup (int N ,pffft_transform_t transform ) { 1250PFFFT_Setup * s = (PFFFT_Setup * )malloc (sizeof (PFFFT_Setup )); 1251int k ,m ; 1252if (!s )return s ; 1253/* unfortunately, the fft size must be a multiple of 16 for complex FFTs 1254and 32 for real FFTs -- a lot of stuff would need to be rewritten to 1255handle other cases (or maybe just switch to a scalar fft, I don't know..) */ 1256if (transform == PFFFT_REAL ) {assert ((N %(2 * SIMD_SZ * SIMD_SZ ))== 0 && N > 0 ); } 1257if (transform == PFFFT_COMPLEX ) {assert ((N %(SIMD_SZ * SIMD_SZ ))== 0 && N > 0 ); } 1258/*assert((N % 32) == 0); */ 1259s -> N = N ; 1260s -> transform = transform ; 1261/* nb of complex simd vectors */ 1262s -> Ncvec = (transform == PFFFT_REAL ?N /2 :N )/SIMD_SZ ; 1263s -> data = (v4sf * )pffft_aligned_malloc (2 * (size_t )s -> Ncvec * sizeof (v4sf )); 1264if (!s -> data ) {free (s );return 0 ;} 1265s -> e = (float * )s -> data ; 1266s -> twiddle = (float * )(s -> data + (2 * s -> Ncvec * (SIMD_SZ - 1 ))/SIMD_SZ ); 1267 1268if (transform == PFFFT_REAL ) { 1269for (k = 0 ;k < s -> Ncvec ;++ k ) { 1270int i = k /SIMD_SZ ; 1271int j = k %SIMD_SZ ; 1272for (m = 0 ;m < SIMD_SZ - 1 ;++ m ) { 1273float A = (float )(-2 * M_PI * (m + 1 )* k /N ); 1274s -> e [(2 * (i * 3 + m )+ 0 )* SIMD_SZ + j ]= cos (A ); 1275s -> e [(2 * (i * 3 + m )+ 1 )* SIMD_SZ + j ]= sin (A ); 1276 } 1277 } 1278rffti1_ps (N /SIMD_SZ ,s -> twiddle ,s -> ifac ); 1279 }else { 1280for (k = 0 ;k < s -> Ncvec ;++ k ) { 1281int i = k /SIMD_SZ ; 1282int j = k %SIMD_SZ ; 1283for (m = 0 ;m < SIMD_SZ - 1 ;++ m ) { 1284float A = (float )(-2 * M_PI * (m + 1 )* k /N ); 1285s -> e [(2 * (i * 3 + m )+ 0 )* SIMD_SZ + j ]= cos (A ); 1286s -> e [(2 * (i * 3 + m )+ 1 )* SIMD_SZ + j ]= sin (A ); 1287 } 1288 } 1289cffti1_ps (N /SIMD_SZ ,s -> twiddle ,s -> ifac ); 1290 } 1291 1292/* check that N is decomposable with allowed prime factors */ 1293for (k = 0 ,m = 1 ;k < s -> ifac [1 ];++ k ) {m *=s -> ifac [2 + k ]; } 1294if (m != N /SIMD_SZ ) { 1295pffft_destroy_setup (s );s = 0 ; 1296 } 1297 1298return s ; 1299} 1300 1301 1302static 1303void pffft_destroy_setup (PFFFT_Setup * s ) { 1304if (!s )return ; 1305pffft_aligned_free (s -> data ); 1306free (s ); 1307} 1308 1309#if !defined(PFFFT_SIMD_DISABLE ) 1310 1311/* [0 0 1 2 3 4 5 6 7 8] -> [0 8 7 6 5 4 3 2 1] */ 1312static void reversed_copy (int N ,const v4sf * in ,int in_stride ,v4sf * out ) { 1313v4sf g0 ,g1 ; 1314int k ; 1315INTERLEAVE2 (in [0 ],in [1 ],g0 ,g1 );in += in_stride ; 1316 1317*-- out = VSWAPHL (g0 ,g1 );/* [g0l, g0h], [g1l g1h] -> [g1l, g0h] */ 1318for (k = 1 ;k < N ;++ k ) { 1319v4sf h0 ,h1 ; 1320INTERLEAVE2 (in [0 ],in [1 ],h0 ,h1 );in += in_stride ; 1321*-- out = VSWAPHL (g1 ,h0 ); 1322*-- out = VSWAPHL (h0 ,h1 ); 1323g1 = h1 ; 1324 } 1325*-- out = VSWAPHL (g1 ,g0 ); 1326} 1327 1328static void unreversed_copy (int N ,const v4sf * in ,v4sf * out ,int out_stride ) { 1329v4sf g0 ,g1 ,h0 ,h1 ; 1330int k ; 1331g0 = g1 = in [0 ];++ in ; 1332for (k = 1 ;k < N ;++ k ) { 1333h0 = * in ++ ;h1 = * in ++ ; 1334g1 = VSWAPHL (g1 ,h0 ); 1335h0 = VSWAPHL (h0 ,h1 ); 1336UNINTERLEAVE2 (h0 ,g1 ,out [0 ],out [1 ]);out += out_stride ; 1337g1 = h1 ; 1338 } 1339h0 = * in ++ ;h1 = g0 ; 1340g1 = VSWAPHL (g1 ,h0 ); 1341h0 = VSWAPHL (h0 ,h1 ); 1342UNINTERLEAVE2 (h0 ,g1 ,out [0 ],out [1 ]); 1343} 1344 1345static 1346void pffft_zreorder (PFFFT_Setup * setup ,const float * in ,float * out ,pffft_direction_t direction ) { 1347int k ,N = setup -> N ,Ncvec = setup -> Ncvec ; 1348const v4sf * vin = (const v4sf * )in ; 1349v4sf * vout = (v4sf * )out ; 1350assert (in != out ); 1351if (setup -> transform == PFFFT_REAL ) { 1352int k ,dk = N /32 ; 1353if (direction == PFFFT_FORWARD ) { 1354for (k = 0 ;k < dk ;++ k ) { 1355INTERLEAVE2 (vin [k * 8 + 0 ],vin [k * 8 + 1 ],vout [2 * (0 * dk + k )+ 0 ],vout [2 * (0 * dk + k )+ 1 ]); 1356INTERLEAVE2 (vin [k * 8 + 4 ],vin [k * 8 + 5 ],vout [2 * (2 * dk + k )+ 0 ],vout [2 * (2 * dk + k )+ 1 ]); 1357 } 1358reversed_copy (dk ,vin + 2 ,8 , (v4sf * )(out + N /2 )); 1359reversed_copy (dk ,vin + 6 ,8 , (v4sf * )(out + N )); 1360 }else { 1361for (k = 0 ;k < dk ;++ k ) { 1362UNINTERLEAVE2 (vin [2 * (0 * dk + k )+ 0 ],vin [2 * (0 * dk + k )+ 1 ],vout [k * 8 + 0 ],vout [k * 8 + 1 ]); 1363UNINTERLEAVE2 (vin [2 * (2 * dk + k )+ 0 ],vin [2 * (2 * dk + k )+ 1 ],vout [k * 8 + 4 ],vout [k * 8 + 5 ]); 1364 } 1365unreversed_copy (dk , (v4sf * )(in + N /4 ), (v4sf * )(out + N - 6 * SIMD_SZ ),-8 ); 1366unreversed_copy (dk , (v4sf * )(in + 3 * N /4 ), (v4sf * )(out + N - 2 * SIMD_SZ ),-8 ); 1367 } 1368 }else { 1369if (direction == PFFFT_FORWARD ) { 1370for (k = 0 ;k < Ncvec ;++ k ) { 1371int kk = (k /4 )+ (k %4 )* (Ncvec /4 ); 1372INTERLEAVE2 (vin [k * 2 ],vin [k * 2 + 1 ],vout [kk * 2 ],vout [kk * 2 + 1 ]); 1373 } 1374 }else { 1375for (k = 0 ;k < Ncvec ;++ k ) { 1376int kk = (k /4 )+ (k %4 )* (Ncvec /4 ); 1377UNINTERLEAVE2 (vin [kk * 2 ],vin [kk * 2 + 1 ],vout [k * 2 ],vout [k * 2 + 1 ]); 1378 } 1379 } 1380 } 1381} 1382 1383static 1384void pffft_cplx_finalize (int Ncvec ,const v4sf * in ,v4sf * out ,const v4sf * e ) { 1385int k ,dk = Ncvec /SIMD_SZ ;/* number of 4x4 matrix blocks */ 1386v4sf r0 ,i0 ,r1 ,i1 ,r2 ,i2 ,r3 ,i3 ; 1387v4sf sr0 ,dr0 ,sr1 ,dr1 ,si0 ,di0 ,si1 ,di1 ; 1388assert (in != out ); 1389for (k = 0 ;k < dk ;++ k ) { 1390r0 = in [8 * k + 0 ];i0 = in [8 * k + 1 ]; 1391r1 = in [8 * k + 2 ];i1 = in [8 * k + 3 ]; 1392r2 = in [8 * k + 4 ];i2 = in [8 * k + 5 ]; 1393r3 = in [8 * k + 6 ];i3 = in [8 * k + 7 ]; 1394VTRANSPOSE4 (r0 ,r1 ,r2 ,r3 ); 1395VTRANSPOSE4 (i0 ,i1 ,i2 ,i3 ); 1396VCPLXMUL (r1 ,i1 ,e [k * 6 + 0 ],e [k * 6 + 1 ]); 1397VCPLXMUL (r2 ,i2 ,e [k * 6 + 2 ],e [k * 6 + 3 ]); 1398VCPLXMUL (r3 ,i3 ,e [k * 6 + 4 ],e [k * 6 + 5 ]); 1399 1400sr0 = VADD (r0 ,r2 );dr0 = VSUB (r0 ,r2 ); 1401sr1 = VADD (r1 ,r3 );dr1 = VSUB (r1 ,r3 ); 1402si0 = VADD (i0 ,i2 );di0 = VSUB (i0 ,i2 ); 1403si1 = VADD (i1 ,i3 );di1 = VSUB (i1 ,i3 ); 1404 1405/* 1406transformation for each column is: 1407 1408[1 1 1 1 0 0 0 0] [r0] 1409[1 0 -1 0 0 -1 0 1] [r1] 1410[1 -1 1 -1 0 0 0 0] [r2] 1411[1 0 -1 0 0 1 0 -1] [r3] 1412[0 0 0 0 1 1 1 1] * [i0] 1413[0 1 0 -1 1 0 -1 0] [i1] 1414[0 0 0 0 1 -1 1 -1] [i2] 1415[0 -1 0 1 1 0 -1 0] [i3] 1416*/ 1417 1418r0 = VADD (sr0 ,sr1 );i0 = VADD (si0 ,si1 ); 1419r1 = VADD (dr0 ,di1 );i1 = VSUB (di0 ,dr1 ); 1420r2 = VSUB (sr0 ,sr1 );i2 = VSUB (si0 ,si1 ); 1421r3 = VSUB (dr0 ,di1 );i3 = VADD (di0 ,dr1 ); 1422 1423* out ++ = r0 ;* out ++ = i0 ;* out ++ = r1 ;* out ++ = i1 ; 1424* out ++ = r2 ;* out ++ = i2 ;* out ++ = r3 ;* out ++ = i3 ; 1425 } 1426} 1427 1428static 1429void pffft_cplx_preprocess (int Ncvec ,const v4sf * in ,v4sf * out ,const v4sf * e ) { 1430int k ,dk = Ncvec /SIMD_SZ ;/* number of 4x4 matrix blocks */ 1431v4sf r0 ,i0 ,r1 ,i1 ,r2 ,i2 ,r3 ,i3 ; 1432v4sf sr0 ,dr0 ,sr1 ,dr1 ,si0 ,di0 ,si1 ,di1 ; 1433assert (in != out ); 1434for (k = 0 ;k < dk ;++ k ) { 1435r0 = in [8 * k + 0 ];i0 = in [8 * k + 1 ]; 1436r1 = in [8 * k + 2 ];i1 = in [8 * k + 3 ]; 1437r2 = in [8 * k + 4 ];i2 = in [8 * k + 5 ]; 1438r3 = in [8 * k + 6 ];i3 = in [8 * k + 7 ]; 1439 1440sr0 = VADD (r0 ,r2 );dr0 = VSUB (r0 ,r2 ); 1441sr1 = VADD (r1 ,r3 );dr1 = VSUB (r1 ,r3 ); 1442si0 = VADD (i0 ,i2 );di0 = VSUB (i0 ,i2 ); 1443si1 = VADD (i1 ,i3 );di1 = VSUB (i1 ,i3 ); 1444 1445r0 = VADD (sr0 ,sr1 );i0 = VADD (si0 ,si1 ); 1446r1 = VSUB (dr0 ,di1 );i1 = VADD (di0 ,dr1 ); 1447r2 = VSUB (sr0 ,sr1 );i2 = VSUB (si0 ,si1 ); 1448r3 = VADD (dr0 ,di1 );i3 = VSUB (di0 ,dr1 ); 1449 1450VCPLXMULCONJ (r1 ,i1 ,e [k * 6 + 0 ],e [k * 6 + 1 ]); 1451VCPLXMULCONJ (r2 ,i2 ,e [k * 6 + 2 ],e [k * 6 + 3 ]); 1452VCPLXMULCONJ (r3 ,i3 ,e [k * 6 + 4 ],e [k * 6 + 5 ]); 1453 1454VTRANSPOSE4 (r0 ,r1 ,r2 ,r3 ); 1455VTRANSPOSE4 (i0 ,i1 ,i2 ,i3 ); 1456 1457* out ++ = r0 ;* out ++ = i0 ;* out ++ = r1 ;* out ++ = i1 ; 1458* out ++ = r2 ;* out ++ = i2 ;* out ++ = r3 ;* out ++ = i3 ; 1459 } 1460} 1461 1462 1463static ALWAYS_INLINE (void )pffft_real_finalize_4x4 (const v4sf * in0 ,const v4sf * in1 ,const v4sf * in , 1464const v4sf * e ,v4sf * out ) { 1465v4sf r0 ,i0 ,r1 ,i1 ,r2 ,i2 ,r3 ,i3 ; 1466v4sf sr0 ,dr0 ,sr1 ,dr1 ,si0 ,di0 ,si1 ,di1 ; 1467r0 = * in0 ;i0 = * in1 ; 1468r1 = * in ++ ;i1 = * in ++ ;r2 = * in ++ ;i2 = * in ++ ;r3 = * in ++ ;i3 = * in ++ ; 1469VTRANSPOSE4 (r0 ,r1 ,r2 ,r3 ); 1470VTRANSPOSE4 (i0 ,i1 ,i2 ,i3 ); 1471 1472/* 1473transformation for each column is: 1474 1475[1 1 1 1 0 0 0 0] [r0] 1476[1 0 -1 0 0 -1 0 1] [r1] 1477[1 0 -1 0 0 1 0 -1] [r2] 1478[1 -1 1 -1 0 0 0 0] [r3] 1479[0 0 0 0 1 1 1 1] * [i0] 1480[0 -1 0 1 -1 0 1 0] [i1] 1481[0 -1 0 1 1 0 -1 0] [i2] 1482[0 0 0 0 -1 1 -1 1] [i3] 1483*/ 1484 1485/*cerr << "matrix initial, before e , REAL:\n 1: " << r0 << "\n 1: " << r1 << "\n 1: " << r2 << "\n 1: " << r3 << "\n"; */ 1486/*cerr << "matrix initial, before e, IMAG :\n 1: " << i0 << "\n 1: " << i1 << "\n 1: " << i2 << "\n 1: " << i3 << "\n"; */ 1487 1488VCPLXMUL (r1 ,i1 ,e [0 ],e [1 ]); 1489VCPLXMUL (r2 ,i2 ,e [2 ],e [3 ]); 1490VCPLXMUL (r3 ,i3 ,e [4 ],e [5 ]); 1491 1492/*cerr << "matrix initial, real part:\n 1: " << r0 << "\n 1: " << r1 << "\n 1: " << r2 << "\n 1: " << r3 << "\n"; */ 1493/*cerr << "matrix initial, imag part:\n 1: " << i0 << "\n 1: " << i1 << "\n 1: " << i2 << "\n 1: " << i3 << "\n"; */ 1494 1495sr0 = VADD (r0 ,r2 );dr0 = VSUB (r0 ,r2 ); 1496sr1 = VADD (r1 ,r3 );dr1 = VSUB (r3 ,r1 ); 1497si0 = VADD (i0 ,i2 );di0 = VSUB (i0 ,i2 ); 1498si1 = VADD (i1 ,i3 );di1 = VSUB (i3 ,i1 ); 1499 1500r0 = VADD (sr0 ,sr1 ); 1501r3 = VSUB (sr0 ,sr1 ); 1502i0 = VADD (si0 ,si1 ); 1503i3 = VSUB (si1 ,si0 ); 1504r1 = VADD (dr0 ,di1 ); 1505r2 = VSUB (dr0 ,di1 ); 1506i1 = VSUB (dr1 ,di0 ); 1507i2 = VADD (dr1 ,di0 ); 1508 1509* out ++ = r0 ; 1510* out ++ = i0 ; 1511* out ++ = r1 ; 1512* out ++ = i1 ; 1513* out ++ = r2 ; 1514* out ++ = i2 ; 1515* out ++ = r3 ; 1516* out ++ = i3 ; 1517 1518} 1519 1520static NEVER_INLINE (void )pffft_real_finalize (int Ncvec ,const v4sf * in ,v4sf * out ,const v4sf * e ) { 1521int k ,dk = Ncvec /SIMD_SZ ;/* number of 4x4 matrix blocks */ 1522/* fftpack order is f0r f1r f1i f2r f2i ... f(n-1)r f(n-1)i f(n)r */ 1523 1524v4sf_union cr ,ci ,* uout = (v4sf_union * )out ; 1525v4sf save = in [7 ],zero = VZERO (); 1526float xr0 ,xi0 ,xr1 ,xi1 ,xr2 ,xi2 ,xr3 ,xi3 ; 1527static const float s = (float )(M_SQRT2 /2 ); 1528 1529cr .v = in [0 ];ci .v = in [Ncvec * 2 - 1 ]; 1530assert (in != out ); 1531pffft_real_finalize_4x4 (& zero ,& zero ,in + 1 ,e ,out ); 1532 1533/* 1534[cr0 cr1 cr2 cr3 ci0 ci1 ci2 ci3] 1535 1536[Xr(1)] ] [1 1 1 1 0 0 0 0] 1537[Xr(N/4) ] [0 0 0 0 1 s 0 -s] 1538[Xr(N/2) ] [1 0 -1 0 0 0 0 0] 1539[Xr(3N/4)] [0 0 0 0 1 -s 0 s] 1540[Xi(1) ] [1 -1 1 -1 0 0 0 0] 1541[Xi(N/4) ] [0 0 0 0 0 -s -1 -s] 1542[Xi(N/2) ] [0 -1 0 1 0 0 0 0] 1543[Xi(3N/4)] [0 0 0 0 0 -s 1 -s] 1544*/ 1545 1546xr0 = (cr .f [0 ]+ cr .f [2 ])+ (cr .f [1 ]+ cr .f [3 ]);uout [0 ].f [0 ]= xr0 ; 1547xi0 = (cr .f [0 ]+ cr .f [2 ])- (cr .f [1 ]+ cr .f [3 ]);uout [1 ].f [0 ]= xi0 ; 1548xr2 = (cr .f [0 ]- cr .f [2 ]);uout [4 ].f [0 ]= xr2 ; 1549xi2 = (cr .f [3 ]- cr .f [1 ]);uout [5 ].f [0 ]= xi2 ; 1550xr1 = ci .f [0 ]+ s * (ci .f [1 ]- ci .f [3 ]);uout [2 ].f [0 ]= xr1 ; 1551xi1 =- ci .f [2 ]- s * (ci .f [1 ]+ ci .f [3 ]);uout [3 ].f [0 ]= xi1 ; 1552xr3 = ci .f [0 ]- s * (ci .f [1 ]- ci .f [3 ]);uout [6 ].f [0 ]= xr3 ; 1553xi3 = ci .f [2 ]- s * (ci .f [1 ]+ ci .f [3 ]);uout [7 ].f [0 ]= xi3 ; 1554 1555for (k = 1 ;k < dk ;++ k ) { 1556v4sf save_next = in [8 * k + 7 ]; 1557pffft_real_finalize_4x4 (& save ,& in [8 * k + 0 ],in + 8 * k + 1 , 1558e + k * 6 ,out + k * 8 ); 1559save = save_next ; 1560 } 1561 1562} 1563 1564static ALWAYS_INLINE (void )pffft_real_preprocess_4x4 (const v4sf * in , 1565const v4sf * e ,v4sf * out ,int first ) { 1566v4sf r0 = in [0 ],i0 = in [1 ],r1 = in [2 ],i1 = in [3 ],r2 = in [4 ],i2 = in [5 ],r3 = in [6 ],i3 = in [7 ]; 1567/* 1568transformation for each column is: 1569 1570[1 1 1 1 0 0 0 0] [r0] 1571[1 0 0 -1 0 -1 -1 0] [r1] 1572[1 -1 -1 1 0 0 0 0] [r2] 1573[1 0 0 -1 0 1 1 0] [r3] 1574[0 0 0 0 1 -1 1 -1] * [i0] 1575[0 -1 1 0 1 0 0 1] [i1] 1576[0 0 0 0 1 1 -1 -1] [i2] 1577[0 1 -1 0 1 0 0 1] [i3] 1578*/ 1579 1580v4sf sr0 = VADD (r0 ,r3 ),dr0 = VSUB (r0 ,r3 ); 1581v4sf sr1 = VADD (r1 ,r2 ),dr1 = VSUB (r1 ,r2 ); 1582v4sf si0 = VADD (i0 ,i3 ),di0 = VSUB (i0 ,i3 ); 1583v4sf si1 = VADD (i1 ,i2 ),di1 = VSUB (i1 ,i2 ); 1584 1585r0 = VADD (sr0 ,sr1 ); 1586r2 = VSUB (sr0 ,sr1 ); 1587r1 = VSUB (dr0 ,si1 ); 1588r3 = VADD (dr0 ,si1 ); 1589i0 = VSUB (di0 ,di1 ); 1590i2 = VADD (di0 ,di1 ); 1591i1 = VSUB (si0 ,dr1 ); 1592i3 = VADD (si0 ,dr1 ); 1593 1594VCPLXMULCONJ (r1 ,i1 ,e [0 ],e [1 ]); 1595VCPLXMULCONJ (r2 ,i2 ,e [2 ],e [3 ]); 1596VCPLXMULCONJ (r3 ,i3 ,e [4 ],e [5 ]); 1597 1598VTRANSPOSE4 (r0 ,r1 ,r2 ,r3 ); 1599VTRANSPOSE4 (i0 ,i1 ,i2 ,i3 ); 1600 1601if (!first ) { 1602* out ++ = r0 ; 1603* out ++ = i0 ; 1604 } 1605* out ++ = r1 ; 1606* out ++ = i1 ; 1607* out ++ = r2 ; 1608* out ++ = i2 ; 1609* out ++ = r3 ; 1610* out ++ = i3 ; 1611} 1612 1613static NEVER_INLINE (void )pffft_real_preprocess (int Ncvec ,const v4sf * in ,v4sf * out ,const v4sf * e ) { 1614int k ,dk = Ncvec /SIMD_SZ ;/* number of 4x4 matrix blocks */ 1615/* fftpack order is f0r f1r f1i f2r f2i ... f(n-1)r f(n-1)i f(n)r */ 1616 1617v4sf_union Xr ,Xi ,* uout = (v4sf_union * )out ; 1618float cr0 ,ci0 ,cr1 ,ci1 ,cr2 ,ci2 ,cr3 ,ci3 ; 1619static const float s = (float )M_SQRT2 ; 1620assert (in != out ); 1621for (k = 0 ;k < 4 ;++ k ) { 1622Xr .f [k ]= ((float * )in )[8 * k ]; 1623Xi .f [k ]= ((float * )in )[8 * k + 4 ]; 1624 } 1625 1626pffft_real_preprocess_4x4 (in ,e ,out + 1 ,1 );/* will write only 6 values */ 1627 1628/* 1629[Xr0 Xr1 Xr2 Xr3 Xi0 Xi1 Xi2 Xi3] 1630 1631[cr0] [1 0 2 0 1 0 0 0] 1632[cr1] [1 0 0 0 -1 0 -2 0] 1633[cr2] [1 0 -2 0 1 0 0 0] 1634[cr3] [1 0 0 0 -1 0 2 0] 1635[ci0] [0 2 0 2 0 0 0 0] 1636[ci1] [0 s 0 -s 0 -s 0 -s] 1637[ci2] [0 0 0 0 0 -2 0 2] 1638[ci3] [0 -s 0 s 0 -s 0 -s] 1639*/ 1640for (k = 1 ;k < dk ;++ k ) { 1641pffft_real_preprocess_4x4 (in + 8 * k ,e + k * 6 ,out - 1 + k * 8 ,0 ); 1642 } 1643 1644cr0 = (Xr .f [0 ]+ Xi .f [0 ])+ 2 * Xr .f [2 ];uout [0 ].f [0 ]= cr0 ; 1645cr1 = (Xr .f [0 ]- Xi .f [0 ])- 2 * Xi .f [2 ];uout [0 ].f [1 ]= cr1 ; 1646cr2 = (Xr .f [0 ]+ Xi .f [0 ])- 2 * Xr .f [2 ];uout [0 ].f [2 ]= cr2 ; 1647cr3 = (Xr .f [0 ]- Xi .f [0 ])+ 2 * Xi .f [2 ];uout [0 ].f [3 ]= cr3 ; 1648ci0 = 2 * (Xr .f [1 ]+ Xr .f [3 ]);uout [2 * Ncvec - 1 ].f [0 ]= ci0 ; 1649ci1 = s * (Xr .f [1 ]- Xr .f [3 ])- s * (Xi .f [1 ]+ Xi .f [3 ]);uout [2 * Ncvec - 1 ].f [1 ]= ci1 ; 1650ci2 = 2 * (Xi .f [3 ]- Xi .f [1 ]);uout [2 * Ncvec - 1 ].f [2 ]= ci2 ; 1651ci3 =- s * (Xr .f [1 ]- Xr .f [3 ])- s * (Xi .f [1 ]+ Xi .f [3 ]);uout [2 * Ncvec - 1 ].f [3 ]= ci3 ; 1652} 1653 1654 1655static 1656void pffft_transform_internal (PFFFT_Setup * setup ,const float * finput ,float * foutput ,v4sf * scratch , 1657pffft_direction_t direction ,int ordered ) { 1658int k ,Ncvec = setup -> Ncvec ; 1659int nf_odd = (setup -> ifac [1 ]& 1 ); 1660 1661#if 0 1662/* temporary buffer is allocated on the stack if the scratch pointer is NULL */ 1663int stack_allocate = (scratch == 0 ?Ncvec * 2 :1 ); 1664VLA_ARRAY_ON_STACK (v4sf ,scratch_on_stack ,stack_allocate ); 1665#endif 1666 1667const v4sf * vinput = (const v4sf * )finput ; 1668v4sf * voutput = (v4sf * )foutput ; 1669v4sf * buff [2 ]; 1670int ib = (nf_odd ^ordered ?1 :0 ); 1671buff [0 ]= voutput ;buff [1 ]= scratch ; 1672 1673assert (VALIGNED (finput )&& VALIGNED (foutput )); 1674 1675/*assert(finput != foutput); */ 1676if (direction == PFFFT_FORWARD ) { 1677ib = !ib ; 1678if (setup -> transform == PFFFT_REAL ) { 1679ib = (rfftf1_ps (Ncvec * 2 ,vinput ,buff [ib ],buff [!ib ], 1680setup -> twiddle ,& setup -> ifac [0 ])== buff [0 ] ?0 :1 ); 1681pffft_real_finalize (Ncvec ,buff [ib ],buff [!ib ], (v4sf * )setup -> e ); 1682 }else { 1683v4sf * tmp = buff [ib ]; 1684for (k = 0 ;k < Ncvec ;++ k ) { 1685UNINTERLEAVE2 (vinput [k * 2 ],vinput [k * 2 + 1 ],tmp [k * 2 ],tmp [k * 2 + 1 ]); 1686 } 1687ib = (cfftf1_ps (Ncvec ,buff [ib ],buff [!ib ],buff [ib ], 1688setup -> twiddle ,& setup -> ifac [0 ],-1 )== buff [0 ] ?0 :1 ); 1689pffft_cplx_finalize (Ncvec ,buff [ib ],buff [!ib ], (v4sf * )setup -> e ); 1690 } 1691if (ordered ) { 1692pffft_zreorder (setup , (float * )buff [!ib ], (float * )buff [ib ],PFFFT_FORWARD ); 1693 }else ib = !ib ; 1694 }else { 1695if (vinput == buff [ib ]) { 1696ib = !ib ;/* may happen when finput == foutput */ 1697 } 1698if (ordered ) { 1699pffft_zreorder (setup , (float * )vinput , (float * )buff [ib ],PFFFT_BACKWARD ); 1700vinput = buff [ib ];ib = !ib ; 1701 } 1702if (setup -> transform == PFFFT_REAL ) { 1703pffft_real_preprocess (Ncvec ,vinput ,buff [ib ], (v4sf * )setup -> e ); 1704ib = (rfftb1_ps (Ncvec * 2 ,buff [ib ],buff [0 ],buff [1 ], 1705setup -> twiddle ,& setup -> ifac [0 ])== buff [0 ] ?0 :1 ); 1706 }else { 1707pffft_cplx_preprocess (Ncvec ,vinput ,buff [ib ], (v4sf * )setup -> e ); 1708ib = (cfftf1_ps (Ncvec ,buff [ib ],buff [0 ],buff [1 ], 1709setup -> twiddle ,& setup -> ifac [0 ],+1 )== buff [0 ] ?0 :1 ); 1710for (k = 0 ;k < Ncvec ;++ k ) { 1711INTERLEAVE2 (buff [ib ][k * 2 ],buff [ib ][k * 2 + 1 ],buff [ib ][k * 2 ],buff [ib ][k * 2 + 1 ]); 1712 } 1713 } 1714 } 1715 1716if (buff [ib ]!= voutput ) { 1717/* extra copy required -- this situation should only happen when finput == foutput */ 1718assert (finput == foutput ); 1719for (k = 0 ;k < Ncvec ;++ k ) { 1720v4sf a = buff [ib ][2 * k ],b = buff [ib ][2 * k + 1 ]; 1721voutput [2 * k ]= a ;voutput [2 * k + 1 ]= b ; 1722 } 1723ib = !ib ; 1724 } 1725assert (buff [ib ]== voutput ); 1726} 1727 1728#if 0 1729void pffft_zconvolve_accumulate (PFFFT_Setup * s ,const float * a ,const float * b ,float * ab ,float scaling ) { 1730int Ncvec = s -> Ncvec ; 1731const v4sf * RESTRICT va = (const v4sf * )a ; 1732const v4sf * RESTRICT vb = (const v4sf * )b ; 1733v4sf * RESTRICT vab = (v4sf * )ab ; 1734 1735#ifdef __arm__ 1736__builtin_prefetch (va ); 1737__builtin_prefetch (vb ); 1738__builtin_prefetch (vab ); 1739__builtin_prefetch (va + 2 ); 1740__builtin_prefetch (vb + 2 ); 1741__builtin_prefetch (vab + 2 ); 1742__builtin_prefetch (va + 4 ); 1743__builtin_prefetch (vb + 4 ); 1744__builtin_prefetch (vab + 4 ); 1745__builtin_prefetch (va + 6 ); 1746__builtin_prefetch (vb + 6 ); 1747__builtin_prefetch (vab + 6 ); 1748# ifndef __clang__ 1749# define ZCONVOLVE_USING_INLINE_NEON_ASM 1750# endif 1751#endif 1752 1753float ar ,ai ,br ,bi ,abr ,abi ; 1754#ifndef ZCONVOLVE_USING_INLINE_ASM 1755v4sf vscal = LD_PS1 (scaling ); 1756int i ; 1757#endif 1758 1759assert (VALIGNED (a )&& VALIGNED (b )&& VALIGNED (ab )); 1760ar = ((v4sf_union * )va )[0 ].f [0 ]; 1761ai = ((v4sf_union * )va )[1 ].f [0 ]; 1762br = ((v4sf_union * )vb )[0 ].f [0 ]; 1763bi = ((v4sf_union * )vb )[1 ].f [0 ]; 1764abr = ((v4sf_union * )vab )[0 ].f [0 ]; 1765abi = ((v4sf_union * )vab )[1 ].f [0 ]; 1766 1767#ifdef ZCONVOLVE_USING_INLINE_ASM /* inline asm version, unfortunately miscompiled by clang 3.2, at least on ubuntu.. so this will be restricted to gcc */ 1768const float * a_ = a ,* b_ = b ;float * ab_ = ab ; 1769int N = Ncvec ; 1770 asmvolatile ("mov r8, %2 \n" 1771"vdup.f32 q15, %4 \n" 1772"1: \n" 1773"pld [%0,#64] \n" 1774"pld [%1,#64] \n" 1775"pld [%2,#64] \n" 1776"pld [%0,#96] \n" 1777"pld [%1,#96] \n" 1778"pld [%2,#96] \n" 1779"vld1.f32 {q0,q1}, [%0,:128]! \n" 1780"vld1.f32 {q4,q5}, [%1,:128]! \n" 1781"vld1.f32 {q2,q3}, [%0,:128]! \n" 1782"vld1.f32 {q6,q7}, [%1,:128]! \n" 1783"vld1.f32 {q8,q9}, [r8,:128]! \n" 1784 1785"vmul.f32 q10, q0, q4 \n" 1786"vmul.f32 q11, q0, q5 \n" 1787"vmul.f32 q12, q2, q6 \n" 1788"vmul.f32 q13, q2, q7 \n" 1789"vmls.f32 q10, q1, q5 \n" 1790"vmla.f32 q11, q1, q4 \n" 1791"vld1.f32 {q0,q1}, [r8,:128]! \n" 1792"vmls.f32 q12, q3, q7 \n" 1793"vmla.f32 q13, q3, q6 \n" 1794"vmla.f32 q8, q10, q15 \n" 1795"vmla.f32 q9, q11, q15 \n" 1796"vmla.f32 q0, q12, q15 \n" 1797"vmla.f32 q1, q13, q15 \n" 1798"vst1.f32 {q8,q9},[%2,:128]! \n" 1799"vst1.f32 {q0,q1},[%2,:128]! \n" 1800"subs %3, #2 \n" 1801"bne 1b \n" 1802 :"+r" (a_ ),"+r" (b_ ),"+r" (ab_ ),"+r" (N ) :"r" (scaling ) :"r8" ,"q0" ,"q1" ,"q2" ,"q3" ,"q4" ,"q5" ,"q6" ,"q7" ,"q8" ,"q9" ,"q10" ,"q11" ,"q12" ,"q13" ,"q15" ,"memory" ); 1803#else /* default routine, works fine for non-arm cpus with current compilers */ 1804for (i = 0 ;i < Ncvec ;i += 2 ) { 1805v4sf ar ,ai ,br ,bi ; 1806ar = va [2 * i + 0 ];ai = va [2 * i + 1 ]; 1807br = vb [2 * i + 0 ];bi = vb [2 * i + 1 ]; 1808VCPLXMUL (ar ,ai ,br ,bi ); 1809vab [2 * i + 0 ]= VMADD (ar ,vscal ,vab [2 * i + 0 ]); 1810vab [2 * i + 1 ]= VMADD (ai ,vscal ,vab [2 * i + 1 ]); 1811ar = va [2 * i + 2 ];ai = va [2 * i + 3 ]; 1812br = vb [2 * i + 2 ];bi = vb [2 * i + 3 ]; 1813VCPLXMUL (ar ,ai ,br ,bi ); 1814vab [2 * i + 2 ]= VMADD (ar ,vscal ,vab [2 * i + 2 ]); 1815vab [2 * i + 3 ]= VMADD (ai ,vscal ,vab [2 * i + 3 ]); 1816 } 1817#endif 1818if (s -> transform == PFFFT_REAL ) { 1819 ((v4sf_union * )vab )[0 ].f [0 ]= abr + ar * br * scaling ; 1820 ((v4sf_union * )vab )[1 ].f [0 ]= abi + ai * bi * scaling ; 1821 } 1822} 1823#endif 1824 1825 1826#else /* defined(PFFFT_SIMD_DISABLE) */ 1827 1828/* standard routine using scalar floats, without SIMD stuff. */ 1829 1830#define pffft_zreorder_nosimd pffft_zreorder 1831static 1832void pffft_zreorder_nosimd (PFFFT_Setup * setup ,const float * in ,float * out ,pffft_direction_t direction ) { 1833int k ,N = setup -> N ; 1834if (setup -> transform == PFFFT_COMPLEX ) { 1835for (k = 0 ;k < 2 * N ;++ k )out [k ]= in [k ]; 1836return ; 1837 } 1838else if (direction == PFFFT_FORWARD ) { 1839float x_N = in [N - 1 ]; 1840for (k = N - 1 ;k > 1 ;-- k )out [k ]= in [k - 1 ]; 1841out [0 ]= in [0 ]; 1842out [1 ]= x_N ; 1843 }else { 1844float x_N = in [1 ]; 1845for (k = 1 ;k < N - 1 ;++ k )out [k ]= in [k + 1 ]; 1846out [0 ]= in [0 ]; 1847out [N - 1 ]= x_N ; 1848 } 1849} 1850 1851#define pffft_transform_internal_nosimd pffft_transform_internal 1852static 1853void pffft_transform_internal_nosimd (PFFFT_Setup * setup ,const float * input ,float * output ,float * scratch , 1854pffft_direction_t direction ,int ordered ) { 1855int Ncvec = setup -> Ncvec ; 1856int nf_odd = (setup -> ifac [1 ]& 1 ); 1857 1858#if 0 1859/* temporary buffer is allocated on the stack if the scratch pointer is NULL */ 1860int stack_allocate = (scratch == 0 ?Ncvec * 2 :1 ); 1861VLA_ARRAY_ON_STACK (v4sf ,scratch_on_stack ,stack_allocate ); 1862#endif 1863float * buff [2 ]; 1864int ib ; 1865/* if (scratch == 0) scratch = scratch_on_stack; */ 1866buff [0 ]= output ;buff [1 ]= scratch ; 1867 1868if (setup -> transform == PFFFT_COMPLEX )ordered = 0 ;/* it is always ordered. */ 1869ib = (nf_odd ^ordered ?1 :0 ); 1870 1871if (direction == PFFFT_FORWARD ) { 1872if (setup -> transform == PFFFT_REAL ) { 1873ib = (rfftf1_ps (Ncvec * 2 ,input ,buff [ib ],buff [!ib ], 1874setup -> twiddle ,& setup -> ifac [0 ])== buff [0 ] ?0 :1 ); 1875 }else { 1876ib = (cfftf1_ps (Ncvec ,input ,buff [ib ],buff [!ib ], 1877setup -> twiddle ,& setup -> ifac [0 ],-1 )== buff [0 ] ?0 :1 ); 1878 } 1879if (ordered ) { 1880pffft_zreorder (setup ,buff [ib ],buff [!ib ],PFFFT_FORWARD );ib = !ib ; 1881 } 1882 }else { 1883if (input == buff [ib ]) { 1884ib = !ib ;/* may happen when finput == foutput */ 1885 } 1886if (ordered ) { 1887pffft_zreorder (setup ,input ,buff [!ib ],PFFFT_BACKWARD ); 1888input = buff [!ib ]; 1889 } 1890if (setup -> transform == PFFFT_REAL ) { 1891ib = (rfftb1_ps (Ncvec * 2 ,input ,buff [ib ],buff [!ib ], 1892setup -> twiddle ,& setup -> ifac [0 ])== buff [0 ] ?0 :1 ); 1893 }else { 1894ib = (cfftf1_ps (Ncvec ,input ,buff [ib ],buff [!ib ], 1895setup -> twiddle ,& setup -> ifac [0 ],+1 )== buff [0 ] ?0 :1 ); 1896 } 1897 } 1898if (buff [ib ]!= output ) { 1899int k ; 1900/* extra copy required -- this situation should happens only when finput == foutput */ 1901assert (input == output ); 1902for (k = 0 ;k < Ncvec ;++ k ) { 1903float a = buff [ib ][2 * k ],b = buff [ib ][2 * k + 1 ]; 1904output [2 * k ]= a ;output [2 * k + 1 ]= b ; 1905 } 1906ib = !ib ; 1907 } 1908assert (buff [ib ]== output ); 1909} 1910 1911#if 0 1912#define pffft_zconvolve_accumulate_nosimd pffft_zconvolve_accumulate 1913void pffft_zconvolve_accumulate_nosimd (PFFFT_Setup * s ,const float * a ,const float * b , 1914float * ab ,float scaling ) { 1915int i ,Ncvec = s -> Ncvec ; 1916 1917if (s -> transform == PFFFT_REAL ) { 1918/* take care of the fftpack ordering */ 1919ab [0 ]+= a [0 ]* b [0 ]* scaling ; 1920ab [2 * Ncvec - 1 ]+= a [2 * Ncvec - 1 ]* b [2 * Ncvec - 1 ]* scaling ; 1921++ ab ;++ a ;++ b ;-- Ncvec ; 1922 } 1923for (i = 0 ;i < Ncvec ;++ i ) { 1924float ar ,ai ,br ,bi ; 1925ar = a [2 * i + 0 ];ai = a [2 * i + 1 ]; 1926br = b [2 * i + 0 ];bi = b [2 * i + 1 ]; 1927VCPLXMUL (ar ,ai ,br ,bi ); 1928ab [2 * i + 0 ]+= ar * scaling ; 1929ab [2 * i + 1 ]+= ai * scaling ; 1930 } 1931} 1932#endif 1933 1934#endif /* defined(PFFFT_SIMD_DISABLE) */ 1935 1936static 1937void pffft_transform (PFFFT_Setup * setup ,const float * input ,float * output ,float * work ,pffft_direction_t direction ) { 1938pffft_transform_internal (setup ,input ,output , (v4sf * )work ,direction ,0 ); 1939} 1940 1941static 1942void pffft_transform_ordered (PFFFT_Setup * setup ,const float * input ,float * output ,float * work ,pffft_direction_t direction ) { 1943pffft_transform_internal (setup ,input ,output , (v4sf * )work ,direction ,1 ); 1944} 1945 1946#endif