<jupyter_start><jupyter_text>Project name here> Summary description here. This file will become your README and also the index of your documentation.  Install `pip install your_project_name`  How to use Fill me in please! Don't forget code examples:<jupyter_code>from fastai.vision.all import *
# torch.cuda.set_device(1)
# df=pd.read_csv('./train.csv')
# imgs=get_files('/home/staff/xin/Downloads/tianchi/train/',extensions='.tif')
# df=pd.DataFrame({'path':imgs})

df=pd.read_csv('same.csv')
df.head()
df=df.drop('path1',1)
df.columns = ['path']
df['valid']=0
df2=df.copy()
df2['valid']=1
df=df.append(df2)
df3=pd.DataFrame({'path':get_files('/home/staff/xin/Downloads/tianchi/teacher/',extensions='.tif')})
df3['valid']=0
df=df.append(df3)
db = DataBlock(blocks=(TransformBlock(type_tfms=partial(RsImage.create)),
                       TransformBlock(type_tfms=partial(RsMask.create)),
                      ),
               get_x=ColReader('path'),
               get_y=ColReader('path'),
               splitter=ColSplitter('valid'),
               item_tfms=[aug,aug2]
              )
# db = DataBlock(blocks=(TransformBlock(type_tfms=partial(RsImage.create)),
#                        TransformBlock(type_tfms=partial(RsMask.create)),
#                       ),
#                get_x=ColReader('path'),
#                get_y=ColReader('path'),
#                splitter=RandomSplitter(valid_pct=0.2,seed=10),
#                item_tfms=[aug,aug2]
#               )

# db.summary(source=df)

dls = db.dataloaders(source=df,bs=32, num_workers=8)

from mmseg.models import build_segmentor
from mmcv.utils import Config
class HRNET(nn.Module):
    def __init__(self,cfgfile):
        super().__init__()
        cfg = Config.fromfile(cfgfile)
        temp=build_segmentor(cfg.model)
        self.backbone =temp.backbone
        self.decode_head = temp.decode_head
    def forward(self, x):
#         x=F.upsample_bilinear(x,scale_factor=2)
        x = self.backbone(x)
        x = self.decode_head(x)
        return x
# loss=OhemCrossEntropy()
def mIOU( pred,label, num_classes=10,scale=4):
    pred =F.upsample_bilinear(pred,scale_factor=scale)  
    pred = F.softmax(pred, dim=1)              
    pred = torch.argmax(pred, dim=1).squeeze(1)
    
    iou_list = list()
    present_iou_list = list()
    pred = pred.view(-1)
    label = label.view(-1)
    # Note: Following for loop goes from 0 to (num_classes-1)
    # and ignore_index is num_classes, thus ignore_index is
    # not considered in computation of IoU.
    for sem_class in range(num_classes):
        if sem_class!=30:
            pred_inds = (pred == sem_class)
            target_inds = (label == sem_class)
            if target_inds.long().sum().item() == 0:
                iou_now = float('nan')
            else: 
                intersection_now = (pred_inds[target_inds]).long().sum().item()
                union_now = pred_inds.long().sum().item() + target_inds.long().sum().item() - intersection_now
                iou_now = float(intersection_now) / float(union_now)
                present_iou_list.append(iou_now)
            iou_list.append(iou_now)
    return np.mean(present_iou_list)
class newloss(nn.Module):
    def __init__(self,):
        super().__init__()
        self.loss = CrossEntropyLossFlat(axis=1)
    def forward(self, input, target):
        return self.loss(RsImage(input),RsImage(target.long()))
class OhemCrossEntropy(nn.Module): 
    def __init__(self, ignore_label=-1, thres=0.7, 
        min_kept=100000, weight=None,scale=4): 
        super(OhemCrossEntropy, self).__init__() 
        self.thresh = thres
        self.min_kept = max(1, min_kept)
        self.ignore_label = ignore_label 
        self.weight=weight
        self.criterion = nn.CrossEntropyLoss(weight=weight, 
                                             ignore_index=ignore_label, 
                                             reduction='none') 
        self.scale=scale
#         self.love=Lovloss()
    
    def forward(self, score, target, **kwargs):
        target=RsImage(target.long())
        score =F.upsample_bilinear(score,scale_factor=self.scale)
        pred = F.softmax(score, dim=1)
        pixel_losses = self.criterion(score, target).contiguous().view(-1)
        mask = target.contiguous().view(-1) != self.ignore_label         
        
        tmp_target = target.clone() 
        tmp_target[tmp_target == self.ignore_label] = 0 
        pred = pred.gather(1, tmp_target.unsqueeze(1)) 
        pred, ind = pred.contiguous().view(-1,)[mask].contiguous().sort()
        min_value = pred[min(self.min_kept, pred.numel() - 1)] 
        threshold = max(min_value, self.thresh) 
        
        pixel_losses = pixel_losses[mask][ind]
        pixel_losses = pixel_losses[pred < threshold] 
        iouloss=Lov.lovasz_softmax(score,target,class_weight=None)
        return pixel_losses.mean()+iouloss/10
class Lovloss(nn.Module): 
    def __init__(self, ignore_label=-1, thres=0.7, 
        min_kept=100000, weight=None): 
        super(Lovloss, self).__init__() 
        self.weight=weight
    
    def forward(self, score, target, **kwargs):
        target=RsImage(target.long())
        score =F.upsample_bilinear(score,scale_factor=4)
        pred = F.softmax(score, dim=1)
        return Lov.lovasz_softmax(score,target,class_weight=self.weight)
weight=torch.FloatTensor([2,1,4,4,2,2,2,4,2,4]).cuda()
loss=OhemCrossEntropy(weight=weight,scale=4)
loss=Lovloss()

model=HRNET(cfgfile='./fcn_hr18.py')
apply_init(model)
learn = Learner(dls,model,metrics=mIOU,loss_func=loss).to_fp16()
learn.load('hrnetdis')
learn.lr_find()
learn.fit_one_cycle(50, 1e-4,cbs=[CSVLogger(fname='hrnetdisiou.csv',append=False),SaveModelCallback(monitor='mIOU',fname='hrnetdisiou')])
learn = Learner(dls,model,metrics=mIOU,loss_func=Lovloss()).to_fp16()
learn.load('hrnetdis')
learn.lr_find()
learn.load('resunet')
stats=torch.load('./models/hrnet.pth')
model.load_state_dict(stats)
imgs=get_files('/home/staff/xin/Downloads/tianchi/test/')
model=model.cuda().eval()
import skimage.io as skio
import tqdm
for img in tqdm.tqdm(imgs):
    temp=RsImage.create(img)
    temp=val_pipe(image=temp.numpy())['image']
    with torch.no_grad():
        result=model(temp.unsqueeze(0).cuda())
    result =F.upsample_nearest(result,scale_factor=4)
    result=result.argmax(axis=1)
    result=result[0]+1
    result=result.cpu().numpy().astype('uint8')
    newname='/home/staff/xin/Downloads/tianchi/resultV0/'+img.name.replace('tif','png')
    skio.imsave(newname,result)
means=[]
for img in tqdm.tqdm(imgs):
    temp=RsImage.create(img)
    temp=val_pipe(image=temp.numpy())['image']
    with torch.no_grad():
        result=model(temp.unsqueeze(0).cuda())
    result =F.upsample_bilinear(result,scale_factor=4)
    result=F.softmax(result)
    mean=result.max(dim=1)[0].mean()
    if mean>0.685:
        result=result.argmax(axis=1)
        result=result[0]+1
        result=result.cpu().numpy().astype('uint8')
        newname='/home/staff/xin/Downloads/tianchi/teacher/'+img.name.replace('tif','png')
        skio.imsave(newname,result)
        shutil.copyfile(str(img),str(img).replace('test','teacher'))
xx=[]
for mean in means:
    xx.append(mean.cpu().item())
xx=np.asarray(xx)
np.count_nonzero(xx>0.685)
result=result.argmax(axis=1)
result=result[0]+1
RsImage.create(img).show()
plt.imshow(result.cpu())
xx=skio.imread('/home/staff/xin/Downloads/tianchi/train/010000.png')
import geopandas as gpd
xx=gpd.read_file('/home/staff/xin/stationbasins.geojson')
xx.to_file('/home/staff/xin/tan/chun')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14906505135633394, 'fraction_numerical': 0.015933631814590465, 'mean_word_length': 3.2217898832684826, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '156490', 'license': 'Apache-2.0', 'n_tokens_mistral': 2741, 'n_tokens_neox': 2657, 'n_words': 361, 'path': 'index-mmdet.ipynb', 'repo_name': 'robeson1010/tianchiclass', 'size': '7594'}