<jupyter_start><jupyter_text>提取特征保存为图片<jupyter_code>SR = 65535 * 2
NUM_CLASSES = 30
LABELS = os.listdir(train_path)
def mono_to_color(
    X: np.ndarray, mean=None, std=None, norm_max=None, norm_min=None, eps=1e-6
):
    """
    Sources:
        https://www.kaggle.com/daisukelab/creating-fat2019-preprocessed-data
        https://www.kaggle.com/ttahara/training-birdsong-baseline-resnest50-fast
    """
    # Stack X as [X,X,X]
    X = np.stack([X, X, X], axis=-1)

    # Standardize
    mean = mean or X.mean()
    X = X - mean
    std = std or X.std()
    Xstd = X / (std + eps)
    _min, _max = Xstd.min(), Xstd.max()
    norm_max = norm_max or _max
    norm_min = norm_min or _min
    if (_max - _min) > eps:
        # Normalize to [0, 255]
        V = Xstd
        V[V < norm_min] = norm_min
        V[V > norm_max] = norm_max
        V = 255 * (V - norm_min) / (norm_max - norm_min)
        V = V.astype(np.uint8)
    else:
        # Just zero
        V = np.zeros_like(Xstd, dtype=np.uint8)
    return V


def normalize(image, mean=None, std=None):
    image = image / 255.0
    if mean is not None and std is not None:
        image = (image - mean) / std
    return np.moveaxis(image, 2, 0).astype(np.float32)


def get_melspec(x, sr, n_mels=256):
    mel_spec = lb.feature.melspectrogram(x, sr=sr, n_mels=n_mels, n_fft=2048, hop_length=512, power=2)
    mel_spec = lb.power_to_db(mel_spec).astype(np.float32)
    image = mono_to_color(mel_spec)
#     image = normalize(image, mean=None, std=None)
    image = image.transpose(2, 1, 0)
#     image = resize(image, (224, 400)) 
    return image


def crop_or_pad(y, length):
    if len(y) < length:
        y = np.concatenate([y, np.zeros(length - len(y))])
    elif len(y) > length:
        y = y[: length]
    return y

def preprocess_train(train_path):
    x, y = [], []
    for i, label in enumerate(LABELS):
        label_dir = os.path.join(train_path, label)
        for wav_file in tqdm(os.listdir(label_dir)):
            wav_path = os.path.join(train_path, label, wav_file)
            wav, sr = lb.load(wav_path, sr=SR)
            wav = crop_or_pad(wav, 1*SR)
            melspec = get_melspec(wav, sr)
            x.append(melspec)
            y.append(i)

    x, y = np.r_[x], np.r_[y]
#     x, y = shuffle(x, y, random_state=GLOBAL_SEED)
    return x, y.astype(np.int32)

def preprocess_test(test_path):
    x, keys = [], []

    for wav_file in tqdm(os.listdir(test_path)):
        wav_path = os.path.join(test_path, wav_file)
        wav, sr = lb.load(wav_path, sr=SR)
        wav = crop_or_pad(wav, 1*SR)
        melspec = get_melspec(wav, sr)
        x.append(melspec)
        keys.append(wav_file)
    x = np.r_[x]
    return x, keys
X_train, y_train = preprocess_train(train_path)
X_test, test_names = preprocess_test(test_path)
np.savez(os.path.join(feat_path, 'melspec_256_256'), X=np.concatenate([X_train, X_test], axis=0), y=y_train, test_names=test_names)<jupyter_output><empty_output><jupyter_text>加载特征准备数据集<jupyter_code>melspec = np.load(os.path.join(feat_path, 'melspec_256_256.npz'))
X = melspec['X']
X_train = X[:57886] 
X_test = X[57886:]
y_train = melspec['y']
test_names = melspec['test_names']
labels = os.listdir(train_path)
del melspec, X
gc.collect()<jupyter_output><empty_output><jupyter_text>准备数据集<jupyter_code>from torchvision import transforms
class CustomDataset(Dataset):
    def __init__(self, index, is_train=True):
        self.index = index
        self.is_train = is_train
        self.transformer = transforms.Compose([
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ])
    
    def __len__(self):
        return len(self.index)
    
    def __getitem__(self, idx):
        if self.is_train:
            img = X_train[self.index[idx]] / 255.0
            return self.transformer(torch.tensor(img, dtype=torch.float32)), y_train[self.index[idx]]
        else:
            img = X_test[self.index[idx]] / 255.0
            return self.transformer(torch.tensor(img, dtype=torch.float32))
BATCH_SIZE_TRAIN = 64
BATCH_SIZE_VAL = 64
BATCH_SIZE_TEST = 64
kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=GLOBAL_SEED)
data_folds = []
valid_indexs = []    


test_dataset = CustomDataset(np.arange(X_test.shape[0]), is_train=False)
test_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE_TEST, sampler=SequentialSampler(test_dataset), shuffle=False, num_workers=0)

for idx, (train_index, valid_index) in enumerate(kf.split(X=X_train, y=y_train)):
    valid_indexs.append(valid_index)
    
    train_dataset = CustomDataset(train_index, is_train=True)
    val_dataset = CustomDataset(valid_index, is_train=True)

    train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE_TRAIN, shuffle=True, num_workers=0)
    valid_dataloader = DataLoader(val_dataset, batch_size=BATCH_SIZE_VAL, sampler=SequentialSampler(val_dataset), shuffle=False, num_workers=0)
    data_folds.append((train_dataloader, valid_dataloader, test_dataloader))<jupyter_output><empty_output><jupyter_text>搭建模型<jupyter_code>import torchvision
class DenseNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.model = torchvision.models.densenet161(pretrained=True)
        self.model.classifier = nn.Linear(2208, 30)    
        
        
    def forward(self, X):
        return self.model(X)
def validate(model, val_dataloader, criterion, history, n_iters):
    model.eval()
    costs = []
    accs = []
    with torch.no_grad():
        for idx, batch in enumerate(val_dataloader):
            X, y = batch
            X, y = X.cuda(), y.cuda().long()
            y_output = model(X)    
            loss = criterion(y_output, y)
            costs.append(loss.item())
            _, y_preds = torch.max(y_output, 1)
            accs.append((y_preds == y).float().mean().item())
    mean_accs = np.mean(accs)
    mean_costs = np.mean(costs)
    writer.add_scalar('age/validate_accuracy', mean_accs, n_iters)
    writer.add_scalar('age/validate_loss', mean_costs, n_iters)
    if mean_accs > history['best_model'][0][0]:  
        heapq.heapify(history['best_model'])
        checkpoint_pth = history['best_model'][0][1]
        heapq.heappushpop(history['best_model'], (mean_accs, checkpoint_pth))
        torch.save(model.state_dict(), checkpoint_pth)
    return mean_costs, mean_accs


def train(model, train_dataloader, val_dataloader, criterion, optimizer, epoch, history, validate_points, scheduler, step=True):
    model.train()
    costs = []
    accs = []
    val_loss, val_acc = 0, 0
    with tqdm(total=len(train_dataloader.dataset), desc='Epoch{}'.format(epoch)) as pbar:
        for idx, batch in enumerate(train_dataloader):
            X, y = batch
            X, y = X.cuda(), y.cuda().long()
            y_output = model(X)    
            loss = criterion(y_output, y)
            
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if step:
                scheduler.step()
            with torch.no_grad():
                costs.append(loss.item())
                _, y_preds = torch.max(y_output, 1)
                accs.append((y_preds == y).float().mean().item())
                pbar.update(y.size(0))
            n_iters = idx + len(train_dataloader) * (epoch-1)
            if idx in validate_points:
                val_loss, val_acc = validate(model, val_dataloader, criterion, history, n_iters)
                model.train()
            
            writer.add_scalar('age/train_accuracy', accs[-1], n_iters)
            writer.add_scalar('age/train_loss', costs[-1], n_iters)
            writer.add_scalar('age/learning_rate', scheduler.get_lr()[0], n_iters)
            pbar.set_postfix_str('loss:{:.4f}, acc:{:.4f}, val-loss:{:.4f}, val-acc:{:.4f}'.format(np.mean(costs[-10:]), np.mean(accs[-10:]), val_loss, val_acc))
            torch.cuda.empty_cache()

    
def test(oof_train_test, model, test_dataloader, val_dataloader, valid_index, weight=1):
    model.eval()
    y_preds = []
    y_preds_val = []
    with torch.no_grad():
        for idx, batch in enumerate(test_dataloader):
            X = batch
            X= X.cuda()
            y_output = model(X)    
            y_preds.append(y_output.cpu())
            
        for idx, batch in enumerate(val_dataloader):
            X, y = batch
            X = X.cuda()
            y_output = model(X)
            y_preds_val.append(y_output.cpu())
    
    oof_train_test[valid_index] += F.softmax(torch.cat(y_preds_val)).numpy() * weight
    oof_train_test[57886:] += F.softmax(torch.cat(y_preds)).numpy() * weight
def criterion(y_output, y_true):
    loss = nn.CrossEntropyLoss()(y_output, y_true)
    return loss

res_folds = []
acc_folds = []
model_name = 'densenet161'
best_checkpoint_num = 3
for idx, (train_dataloader, val_dataloader, test_dataloader) in enumerate(data_folds):
    oof_train_test = np.zeros((X_train.shape[0] + X_test.shape[0], 30))
    history = {'best_model': []}
    for i in range(best_checkpoint_num):
        history['best_model'].append((0, os.path.join(model_save, '{}_checkpoint_fold_{}_{}.pth'.format(model_name, idx, i))))
    validate_points = list(np.linspace(0, len(train_dataloader)-1, 3).astype(int))[1:]
     
    model = DenseNet().cuda()
#     model = nn.DataParallel(model, device_ids=[0, 1])
    optimizer = torch.optim.Adam(model.parameters(), betas=(0.9, 0.999), lr=1e-3)
    epochs = 6
#     scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.5)
    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-5, max_lr=2e-3, step_size_up=int(len(train_dataloader)/2), cycle_momentum=False, mode='triangular')
#     scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=3e-3, epochs=epochs, steps_per_epoch=len(train_dataloader), pct_start=0.2, anneal_strategy='linear', div_factor=30, final_div_factor=1e4)
    for epoch in range(1, epochs+1):
        writer = SummaryWriter(log_dir=os.path.join(tensorboard_path, '{}_fold_{}'.format(model_name, idx)))
        train(model, train_dataloader, val_dataloader, criterion, optimizer, epoch, history, validate_points, scheduler, step=True)
#         scheduler.step()
        gc.collect()
    for (acc, checkpoint_pth), weight in zip(sorted(history['best_model'], reverse=True), [0.5, 0.3, 0.2]):
        model.load_state_dict(torch.load(checkpoint_pth, map_location= torch.device('cpu')), strict=True)
        test(oof_train_test, model, test_dataloader, val_dataloader, valid_indexs[idx], weight=weight)
    acc_folds.append(sorted(history['best_model'], reverse=True)[0][0])
    res_folds.append(oof_train_test)
    np.save(os.path.join(res_path, "{}_fold_{}.npy".format(model_name, idx)), oof_train_test)
    del model, history 
    gc.collect()
    torch.cuda.empty_cache()
acc_folds
res = []
for i in range(len(data_folds)):
    res.append(np.load(os.path.join(res_path, "{}_fold_{}.npy".format(model_name, i))))
len(res)
sub = pd.DataFrame()
sub['file_name'] = test_names
sub['label'] = np.argmax(np.mean(res, axis=0)[57886:], axis=1)
sub['label'] = sub['label'].map({i:label for i, label in enumerate(labels)})
sub
now = time.strftime("%Y%m%d_%H%M%S",time.localtime(time.time())) 
fname="submit_" + model_name + "_" + now + ".csv"    
sub.to_csv(os.path.join(res_path, fname), index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14143724162195442, 'fraction_numerical': 0.019614741841850646, 'mean_word_length': 3.233060312732688, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '180981', 'license': 'MIT', 'n_tokens_mistral': 4434, 'n_tokens_neox': 4252, 'n_words': 722, 'path': 'ccf_audio/Densenet baseline.ipynb', 'repo_name': 'PerpetualSmile/Code-Repository', 'size': '11369'}