<commit_before>#!/usr/local/bin/python2.3
import math
import sys
import os
import random
import struct
import popen2
import getopt
import Numeric
import FFT

pi=math.pi
e=math.e
j=complex(0,1)

doreal=0

datatype = os.environ.get('DATATYPE','float')

util = '../tools/fft'
fmt='f'
minsnr=90
if datatype == 'double':
    util = '../tools/fft_double'
    fmt='d'
elif datatype=='short':
    util = '../tools/fft_short'
    fmt='h'
    minsnr=10


def dopack(x,cpx=1):
    x = Numeric.reshape( x, ( Numeric.size(x),) )
    if cpx:
        s = ''.join( [ struct.pack(fmt*2,c.real,c.imag) for c in x ] )
    else:
        s = ''.join( [ struct.pack(fmt,c) for c in x ] )
    return s

def dounpack(x,cpx):
    uf = fmt * ( len(x) / struct.calcsize(fmt) )
    s = struct.unpack(uf,x)
    if cpx:
        return Numeric.array(s[::2]) + Numeric.array( s[1::2] )*j
    else:
        return Numeric.array(s )

def make_random(dims=[1]):
    res = []
    for i in range(dims[0]):
        if len(dims)==1:
            r=random.uniform(-1,1)
            if doreal:
                res.append( r )
            else:
                i=random.uniform(-1,1)
                res.append( complex(r,i) )
        else:
            res.append( make_random( dims[1:] ) )
    return Numeric.array(res)

def flatten(x):
    ntotal = Numeric.product(Numeric.shape(x))
    return Numeric.reshape(x,(ntotal,))

def randmat( ndims ):
    dims=[]
    for i in range( ndims ):
        curdim = int( random.uniform(2,4) )
        dims.append( curdim )
    return make_random(dims )

def test_fft(ndims):
    if ndims == 1:
        nfft = int(random.uniform(50,520))
        if doreal:
            nfft = int(nfft/2)*2

        x = Numeric.array(make_random( [ nfft ] ) )
    else:
        x=randmat( ndims )

    print 'dimensions=%s' % str( Numeric.shape(x) ),
    if doreal:
        xver = FFT.real_fft(x)
    else:
        xver = FFT.fftnd(x)
    
    x2=dofft(x)
    err = xver - x2
    errf = flatten(err)
    xverf = flatten(xver)
    errpow = Numeric.vdot(errf,errf)+1e-10
    sigpow = Numeric.vdot(xverf,xverf)+1e-10
    snr = 10*math.log10(abs(sigpow/errpow) )
    print 'SNR (compared to NumPy) : %.1fdB' % float(snr)

    if snr<minsnr:
        print 'xver=',xver
        print 'x2=',x2
        print 'err',err
        sys.exit(1)
 
def dofft(x):<commit_msg>real_fftnd can do everything real_fftnd can do<commit_after>#!/usr/local/bin/python2.3
import math
import sys
import os
import random
import struct
import popen2
import getopt
import Numeric
import FFT

pi=math.pi
e=math.e
j=complex(0,1)

doreal=0

datatype = os.environ.get('DATATYPE','float')

util = '../tools/fft'
fmt='f'
minsnr=90
if datatype == 'double':
    util = '../tools/fft_double'
    fmt='d'
elif datatype=='short':
    util = '../tools/fft_short'
    fmt='h'
    minsnr=10


def dopack(x,cpx=1):
    x = Numeric.reshape( x, ( Numeric.size(x),) )
    if cpx:
        s = ''.join( [ struct.pack(fmt*2,c.real,c.imag) for c in x ] )
    else:
        s = ''.join( [ struct.pack(fmt,c) for c in x ] )
    return s

def dounpack(x,cpx):
    uf = fmt * ( len(x) / struct.calcsize(fmt) )
    s = struct.unpack(uf,x)
    if cpx:
        return Numeric.array(s[::2]) + Numeric.array( s[1::2] )*j
    else:
        return Numeric.array(s )

def make_random(dims=[1]):
    res = []
    for i in range(dims[0]):
        if len(dims)==1:
            r=random.uniform(-1,1)
            if doreal:
                res.append( r )
            else:
                i=random.uniform(-1,1)
                res.append( complex(r,i) )
        else:
            res.append( make_random( dims[1:] ) )
    return Numeric.array(res)

def flatten(x):
    ntotal = Numeric.product(Numeric.shape(x))
    return Numeric.reshape(x,(ntotal,))

def randmat( ndims ):
    dims=[]
    for i in range( ndims ):
        curdim = int( random.uniform(2,4) )
        dims.append( curdim )
    return make_random(dims )

def test_fft(ndims):
    if ndims == 1:
        nfft = int(random.uniform(50,520))
        if doreal:
            nfft = int(nfft/2)*2

        x = Numeric.array(make_random( [ nfft ] ) )
    else:
        x=randmat( ndims )

    print 'dimensions=%s' % str( Numeric.shape(x) ),
    if doreal:
        xver = FFT.real_fftnd(x)
    else:
        xver = FFT.fftnd(x)
    
    x2=dofft(x)
    err = xver - x2
    errf = flatten(err)
    xverf = flatten(xver)
    errpow = Numeric.vdot(errf,errf)+1e-10
    sigpow = Numeric.vdot(xverf,xverf)+1e-10
    snr = 10*math.log10(abs(sigpow/errpow) )
    print 'SNR (compared to NumPy) : %.1fdB' % float(snr)

    if snr<minsnr:
        print 'xver=',xver
        print 'x2=',x2
        print 'err',err
        sys.exit(1)
 
def dofft(x):
{'fraction_non_alphanumeric': 0.15417286047993203, 'fraction_numerical': 0.02081121257167127, 'mean_word_length': 2.232669869595058, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5676574', 'n_tokens_mistral': 1938, 'n_tokens_neox': 1897, 'n_words': 385}