<jupyter_start><jupyter_text>Polar representation of CQT SpectrogramsConstant Q transforms (CQT) uses longer frames on lower frequencies and shorter on higher (As far as i understand it).  This means that we have a dynamic spectral/time resolution, and therefore do not loose as much information as with normal STFTs.  HypothesisLower frequencies changes less often than higher.  That means that the amount of inforation grows with frequency.  If higher frequencies get more space (pixels) and and lower frequencies gets less,  one can assume the information density in each spectrogram is optimal.   One way to to achieve this is wraping the rectangular images to a circle,  such that lower friequencies will be in the middle of that circle, and higher on the edges.  This gives higher frequencies more space as the circumference of a circle grows with its radius.    Gather data<jupyter_code>path = Path('../dataset/DeepShip')
sr = 32000
fns =  get_files(path,'.wav')
fns
# Figure out fft sizes<jupyter_output><empty_output><jupyter_text>Reading wav, making and normalizing spectrograms<jupyter_code>def normSamp(audio):
    ret = audio - torch.mean(audio)
    return ret / torch.max(ret)

def normSpec(spec):
    # take the logarithm of the values
    ret = torch.log10(spec+1e-20)
    mean = torch.mean(ret)
    std = torch.std(ret)
    # Normalize each frame so its max 1, we dont need the extra dimension
    #return (ret / torch.transpose(torch.max(ret,2)[0],0,1))[0]
    #return (ret / torch.max(ret))[0]
    
    ret =  (ret - mean) / (std*4) + 0.5
    return torch.clamp(ret, min=0, max=1) 

def normCqt(cqt):
    ret = torch.log10(cqt) / 2
    #return (ret / torch.max(ret,1)[0])[0]
    ret = (ret - torch.mean(ret) + 0.5)
    return torch.clamp(ret, min=0, max=1) 

def readWav(p: Path, num_frames = -1, rand=True):
    frames = torchaudio.info(p).num_frames
    last_start = frames-num_frames if num_frames >0  else frames

    # If enough frames, return selection
    if last_start > 0: 
        # Random start point or middle of file
        start = random.randint(0,last_start) if rand else int(last_start/2)
        return torchaudio.load(p, num_frames=num_frames, frame_offset=start)[0]
    # if not enough frames in audioclip, return everything
    else: 
        return torchaudio.load(p)[0]<jupyter_output><empty_output><jupyter_text>The spectogram transforms<jupyter_code>#  CQT
img_size = 224

hop_lf = sr//4
rng_lf = hop_lf * ( img_size - 1 )
cqt_lf = features.CQT(sr=sr,hop_length=hop_lf, bins_per_octave=51, fmin=8, fmax=3500, verbose=False).cuda()

nfft_hf = 512
hop_hf = int(nfft_hf*0.9)
rng_hf = hop_hf * ( img_size - 1 ) #+ nfft_hf
stft_hf = features.STFT(sr=sr,n_fft=nfft_hf,hop_length=hop_hf,output_format='Magnitude',verbose=False).cuda()
#features.STFT?
class Spectrogram(TensorImageBase):
    """Type to represent a spectogram which knows show itself"""
    @classmethod
    def create(cls, o: Tensor):
        return cls(o)
    
    def show(self, figsize=None, ctx=None, **kwargs): 
        channels = self.shape[0]
        t = self
        if not isinstance(t, Tensor): return ctx 
        if figsize is None: figsize=(10,10)   
        #f, axarr = plt.subplots(1,3,figsize=(15,15))
        #axarr[0].imshow(specs[0,:,:].cpu(),extent=[0,imgsize,(imgsize+1)*Fbin_lf,Fbin_lf] ,aspect=1/Fbin_lf)
        #axarr[1].imshow(specs[1,:,:].cpu(),extent=[0,imgsize,(imgsize+50)*Fbin_hf,Fbin_hf*50] ,aspect=1/Fbin_hf)
        #axarr[2].imshow(specs[2,:,:].cpu())
        return show_images(t, nrows=1, ncols=channels)
def wavToSpecs(wavs : torch.Tensor, hf_idx=0, pad_random=True):
    img = cqt_lf(wavs[0,:rng_lf])[0]
    
    img_len = img.shape[1]
    if img_len < img_size:
        pad = img_size-img_len
        
        # Reflect Padding
        if pad > img_len: 
            pad_left = random.randint(0,img_len-1) if pad_random else img_len -1
            pad_right = random.randint(0,img_len-1) if pad_random else img_len -1
        else:     
            pad_left = random.randint(0,pad) if pad_random else pad//2
            pad_right = pad - pad_left;
            
        img = torch.nn.functional.pad(img,(pad_left,pad_right),'reflect')
        img = img[:,0:img_size]
        .
        # Repeat padding
        img_len = img.shape[1]
        if (img_len < img_size):
            repeat = math.ceil(img_size / img_len)
            for i in range(repeat):
                if i%2==0:
                    img = torch.cat((img,torch.fliplr(img)),1)
                else:
                    img = torch.cat((img,img),1)    
            #img = img.repeat(1,repeat)
            start = random.randint(0,img_len*repeat-img_size) if pad_random else 0
            img = img[:,start:start+img_size]
            
    lf = normCqt(img[0:img_size])
    mf = normCqt(img[img_size:img_size*2])
    .
    img = stft_hf(wavs[0,hf_idx:hf_idx+rng_hf])[0]
    hf = normSpec(img[13:-20])
    return Spectrogram.create(torch.stack((lf, mf, hf),0))<jupyter_output><empty_output><jupyter_text>Testing transforms<jupyter_code>file = fns[33]
wav = readWav(file, rng_lf, True).cuda()
img = wavToSpecs(wav,0, True)
img.show()<jupyter_output><empty_output><jupyter_text>The transform<jupyter_code>#norm = T.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
def label_func(p : Path):
    if PurePath(p).parent.name == "AmbientSE": return []
    return [PurePath(p).parent.parent.name]
label_func(fns[245])
#show_image??
# Make a fastai Transform
class SpectrogramTransform(RandTransform):
    "A transform handler for multiple `spect` transforms"
    split_idx,order=None,0  # 0 = HIGH prio
    #def __init__(self, train_aug, valid_aug): store_attr()
    def __init__(self): 
        store_attr()
    
    def before_call(self, b, split_idx):
        self.idx = split_idx
    
    def encodes(self, p : Path):
        
        if self.idx == 0: #Train transform
            wav = readWav(p, rng_lf, True)
            hf_idx = random.randint(0,wav.shape[1]-rng_hf)
            return wavToSpecs(wav.cuda(), hf_idx, True)
        else: #Valid transform
            wav = readWav(p, rng_lf, False)
            hf_idx = (wav.shape[1]-rng_hf) //2
            return wavToSpecs(wav.cuda(), hf_idx, False)
    
    #def decodes(self, x): return TitledImage(x,'test')
def get_wavs(p : Path) :
    l = get_files(p,'.wav')
    ret = L()
    for f in l:
        frames = torchaudio.info(f).num_frames
        if frames*2 > rng_lf:
            ret.append(f)
    return ret
#class ImgTransform(ItemTransform):
#    def __init__(self, vocab): self.vocab = vocab
#    def encodes(self, o): return o
    #def decodes(self, x): return TitledImage(x[0],self.vocab[x[1]])
class PolarTransform(ItemTransform):
    split_idx = None
    
    def encodes(self, tensor : torch.Tensor):
        # Rearrange and move to cpu
        transposed = torch.transpose(img*255,1,2).permute(1,2,0).cpu().numpy()
        height, width = transposed.shape[:2]
        center = (height/2, width/2)
        radius = np.sqrt(((height/2.0)**2.0)+((width/2.0)**2.0))
        polar_image = cv2.linearPolar(transposed, center, radius, cv2.WARP_FILL_OUTLIERS+cv2.WARP_INVERSE_MAP)
        return torch.from_numpy(np.swapaxes(polar_image,0, 2))
    
    
p = PolarTransform
test  = p.encodes(img)
plt.imshow(test[1])
plt.imshow(test[2])
plt.imshow(test[0])<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>db = DataBlock(blocks=(TransformBlock(type_tfms=SpectrogramTransform),  MultiCategoryBlock),
               get_items = get_wavs,
               get_y = label_func,
               splitter = RandomSplitter(valid_pct=0.2, seed=42),
               item_tfms=PolarTransform
             )
dls = db.dataloaders(path, bs=16, num_workers=0)
dls.vocab<jupyter_output><empty_output><jupyter_text>Experimenting with different optimization functions<jupyter_code>learn = cnn_learner(dls, resnet50,  n_in=3, pretrained=True,
                    loss_func=BCEWithLogitsLossFlat(),
                    opt_func = ranger,  # partial(OptimWrapper, opt=torch.optim.SGD)
                    metrics=[accuracy_multi]).to_fp16() #partial(accuracy_multi, thresh=0.95)
lr = learn.lr_find()
#SaveModelCallback?
cbs = [SaveModelCallback(fname='polar',monitor='accuracy_multi')]
learn.fine_tune(200,base_lr=1e-3, cbs=cbs, MixUp())
#learn.fit_flat_cos(2, lr=6e-3)
learn.unfreeze()
learn.lr_find()
learn.fit_flat_cos(100, lr=7e-4)
learn.lr_find()
learn.fine_tune(100,base_lr=7e-6, cbs=cbs)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12681031437654539, 'fraction_numerical': 0.025197221241022017, 'mean_word_length': 3.274786109713135, 'pattern_counts': {'":': 1, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '524729', 'license': 'MIT', 'n_tokens_mistral': 3211, 'n_tokens_neox': 3070, 'n_words': 653, 'path': 'Experiments/8. deepship_3ch_qct_polar.ipynb', 'repo_name': 'voibit/dat255', 'size': '8493'}