<jupyter_start><jupyter_text>sigMF STFT on GPU and CPU<jupyter_code>import os
import itertools
from sklearn.utils import shuffle
import torch, torchvision
import torch.nn as nn
import torch.nn.functional as d
import torch.optim as optim
import torch.nn.functional as F
import torch.nn.modules as mod
import torch.utils.data
import torch.utils.data as data
from torch.nn.utils.rnn import pack_padded_sequence
from torch.nn.utils.rnn import pad_packed_sequence
from torch.autograd import Variable
import numpy as np
import sys
import importlib
import time
import matplotlib.pyplot as plt
import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torchvision.utils import save_image
import librosa
from scipy import signal
from scipy import stats
from scipy.special import comb
import matplotlib.pyplot as plt
import glob
import json
import pickle
from random import randint, choice
import random
from timeit import default_timer as timer
# from torchaudio.functional import istft
from torch import istft
from sklearn.decomposition import NMF
plt.style.use('default')
device = torch.device('cuda:0')
print('Torch version =', torch.__version__, 'CUDA version =', torch.version.cuda)
print('CUDA Device:', device)
print('Is cuda available? =',torch.cuda.is_available())
# %matplotlib notebook
# %matplotlib inline<jupyter_output><empty_output><jupyter_text>Machine paths<jupyter_code>path_save = "/home/david/sigMF_ML/SVD/saved/" 
path = "/home/david/sigMF_ML/SVD/" 
print(path)<jupyter_output>/home/david/sigMF_ML/SVD/<jupyter_text>reading sigmf meta data and encoder function<jupyter_code># START OF FUNCTIONS ****************************************************
def meta_encoder(meta_list, num_classes): 
    a = np.asarray(meta_list, dtype=int)
#    print('a = ', a)
    return a 

def read_meta(meta_files):
    meta_list = []
    for meta in meta_files:
        all_meta_data = json.load(open(meta))
        meta_list.append(all_meta_data['global']["core:class"])
    meta_list = list(map(int, meta_list))
    return meta_list

def read_num_val(x):
    x = len(meta_list_val)
    return x
print(path)
os.chdir(path)
data_files = sorted(glob.glob('*.sigmf-data'))
meta_files = sorted(glob.glob('*.sigmf-meta'))
for meta in meta_files:
    all_meta_data = json.load(open(meta))
    print("file name = ", meta)<jupyter_output>file name =  UHF_UV5R_V2_plus_voice__snr_hi.sigmf-meta<jupyter_text>torch GPU Cuda stft<jupyter_code>def gpu(db, n_fft):
    I = db[0::2]
    Q = db[1::2]
    start = timer()
    w = n_fft
    win = torch.hann_window(w, periodic=True, dtype=None, layout=torch.strided, requires_grad=False).cuda()
    I_stft = torch.stft(torch.tensor(I).cuda(), n_fft=n_fft, hop_length=n_fft//2, win_length=w, window=win, center=True, normalized=True, onesided=False)
    Q_stft = torch.stft(torch.tensor(Q).cuda(), n_fft=n_fft, hop_length=n_fft//2, win_length=w, window=win, center=True, normalized=True, onesided=False)
    X_stft = I_stft[...,0] + Q_stft[...,0] + I_stft[...,1] + -1*Q_stft[...,1]
    X_stft = torch.cat((X_stft[n_fft//2:],X_stft[:n_fft//2]))
    end = timer() # mag spec of sum of I and Q complex channels; sqrt((a+bj)^2+(c+dj)^2)
    print(end - start)
    torch.cuda.empty_cache()
    return X_stft, I_stft, Q_stft<jupyter_output><empty_output><jupyter_text>scipy CPU stft function<jupyter_code>def cpu(db, n_fft):
    t = len(db)
    db2 = db[0::]
    start = timer()
    db = db.astype(np.float32).view(np.complex64)
    Fs = 1e6
    I_t, I_f, Z = signal.stft(db, fs=Fs, nperseg=n_fft, return_onesided=False)
    Z = np.vstack([Z[n_fft//2:], Z[:n_fft//2]])
    end = timer()
    print(end - start)
    return Z<jupyter_output><empty_output><jupyter_text>GPU Timing: Slow the first time running<jupyter_code>n_fft = 1024
for file in data_files:
    db = np.fromfile(file, dtype="float32")
    X_stft, I_stft, Q_stft = gpu(db, n_fft)
plt.imshow(20*np.log10(np.abs(X_stft.cpu()+1e-8)), aspect='auto', origin='lower')
plt.show()<jupyter_output><empty_output><jupyter_text>CPU Timing<jupyter_code>for file in data_files:
    db = np.fromfile(file, dtype="float32")
    stft_cpu = cpu(db, 1000)<jupyter_output>0.20826889807358384<jupyter_text>CPU load stft to Cuda Time<jupyter_code>start = timer()
IQ_tensor = torch.tensor(np.abs(stft_cpu)).cuda()
end = timer()
print(end - start)
torch.cuda.empty_cache()
plt.imshow(20*np.log10(np.abs(stft_cpu)+1e-8), aspect='auto', origin='lower')
plt.show()<jupyter_output><empty_output><jupyter_text>GPU SVD<jupyter_code>def udv_stft(I_stft,Q_stft):
    start = timer()
    U_I0, D_I0, V_I0 = torch.svd(I_stft[...,0]) 
    U_I1, D_I1, V_I1 = torch.svd(I_stft[...,1]) 
    U_Q0, D_Q0, V_Q0 = torch.svd(Q_stft[...,0]) 
    U_Q1, D_Q1, V_Q1 = torch.svd(Q_stft[...,1]) 
    end = timer()
    print('SVD time: ',end - start)
    return U_I0, D_I0, V_I0, U_I1, D_I1, V_I1, U_Q0, D_Q0, V_Q0, U_Q1, D_Q1, V_Q1<jupyter_output><empty_output><jupyter_text>Inverse stft<jupyter_code># def ISTFT(db, n_fft):# We are matching scipy.signal behavior (setting noverlap=frame_length - hop) 
#     w = 512
#     win = torch.hann_window(w, periodic=True, dtype=None, layout=torch.strided, requires_grad=False).cuda()
#     start = timer()
#     Z = istft(db, n_fft=n_fft, hop_length=n_fft//2, win_length=w, window=win, center=True, normalized=True, onesided=False)
#     end = timer()
#     print('ISTFT time = ',end - start)
#     torch.cuda.empty_cache()
#     return Z
def ISTFT(db, n_fft):# We are matching scipy.signal behavior (setting noverlap=frame_length - hop) 
    w = n_fft
    win = torch.hann_window(w, periodic=True, dtype=None, layout=torch.strided, requires_grad=False).cuda()
    start = timer()
    Z = istft(db, n_fft=n_fft, hop_length=n_fft//2, win_length=n_fft, window=win, center=True, normalized=True, onesided=False)
    end = timer()
    print('ISTFT time = ',end - start)
    torch.cuda.empty_cache()
    return Z<jupyter_output><empty_output><jupyter_text>Re-combine UDV to approximate original signal<jupyter_code>def udv(u, d, v, k): # like ----> np.matrix(U[:, :k]) * np.diag(D[:k]) * V[:k, :]
    start = timer()
    UD = torch.mul(u[:, :k], d[:k])
    v = torch.transpose(v,1,0)
    UDV = torch.mm(UD, v[:k, :])
    end = timer()
    print('UDV time: ',end - start)
    return UDV<jupyter_output><empty_output><jupyter_text>Main function to run all sub function calls<jupyter_code>def complete(I_stft,Q_stft, num, n_fft):
    U_I0, D_I0, V_I0, U_I1, D_I1, V_I1, U_Q0, D_Q0, V_Q0, U_Q1, D_Q1, V_Q1 = udv_stft(I_stft,Q_stft)
    torch.cuda.empty_cache()
    print('UDV I0 shapes = ',U_I0.shape, D_I0.shape, V_I0.shape)
    print('UDV I1 shapes = ',U_I1.shape, D_I1.shape, V_I1.shape)
    print('UDV Q0 shapes = ', U_Q0.shape, D_Q0.shape, V_Q0.shape)
    print('UDV Q1 shapes = ', U_Q1.shape, D_Q1.shape, V_Q1.shape)
    udv_I0 = udv(U_I0, D_I0, V_I0,num)
    udv_I1 = udv(U_I1, D_I1, V_I1,num)
    udv_Q0 = udv(U_Q0, D_Q0, V_Q0,num)
    udv_Q1 = udv(U_Q1, D_Q1, V_Q1,num)
    torch.cuda.empty_cache()
    print('udv I shapes = ',udv_I0.shape,udv_I1.shape)
    print('udv Q shapes = ',udv_Q0.shape,udv_Q1.shape)
    # -------------stack and transpose----------------------------------------
    UDV_I = torch.stack([udv_I0,udv_I1])
    UDV_I = torch.transpose(UDV_I,2,0)
    UDV_I = torch.transpose(UDV_I,1,0)
    UDV_Q = torch.stack([udv_Q0,udv_Q1])
    UDV_Q = torch.transpose(UDV_Q,2,0)
    UDV_Q = torch.transpose(UDV_Q,1,0)
    torch.cuda.empty_cache()
    #--------------------------------------------------------------------------
    I = ISTFT(UDV_I, n_fft)
    Q = ISTFT(UDV_Q, n_fft)
    torch.cuda.empty_cache()
    I = I.detach().cpu().numpy()
    Q = Q.detach().cpu().numpy()
    end = len(I)*2
    IQ_SVD = np.zeros(len(I)*2)
    IQ_SVD[0:end:2] = I
    IQ_SVD[1:end:2] = Q 
    IQ_SVD = IQ_SVD.astype(np.float32).view(np.complex64)
    return IQ_SVD<jupyter_output><empty_output><jupyter_text>Perform SVD on IQ stft data<jupyter_code>num = 3 # number to reconstruct SVD matrix from
IQ_SVD = complete(I_stft,Q_stft, num, n_fft)
torch.cuda.empty_cache()<jupyter_output><empty_output><jupyter_text>Write reconstructed IQ file to file<jupyter_code>from array import array
IQ_file = open("UV5R_voice2", 'wb')
IQ_SVD.tofile(IQ_file)
IQ_file.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16196333815726002, 'fraction_numerical': 0.021466473709599615, 'mean_word_length': 4.282165605095542, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '6108', 'license': 'MIT', 'n_tokens_mistral': 3508, 'n_tokens_neox': 3435, 'n_words': 649, 'path': 'sigMF GPU STFT SVD voice reconstruction v6 old.ipynb', 'repo_name': 'rdbadger/RF_SVD', 'size': '8292'}