<jupyter_start><jupyter_text>SmoteR AlgorithmHere we use the forest fires dataset from Kaggle and the notebook from https://www.kaggle.com/aleksandradeis/regression-addressing-extreme-rare-cases<jupyter_code>import pandas as pd

fires = pd.read_csv("/Users/mling/OutSource/test/data/forestfires.csv")
fires.head()
fires = fires.drop(columns=['X', 'Y'])

fires.groupby('month').agg('count')
months_to_remove = ['nov', 'jan', 'may']

fires[fires['month'].isin(months_to_remove)]<jupyter_output><empty_output><jupyter_text>1. Drop months ['nov', 'jan', 'may']2. One-hot encoding for categorical variables month and days for KNN algorithm<jupyter_code>fires = fires.drop(fires[fires.month.isin(months_to_remove)].index ,axis=0)

fires = pd.get_dummies(fires)<jupyter_output><empty_output><jupyter_text>check the target variable 'area' distribution<jupyter_code>import matplotlib.pyplot as plt

plt.hist(fires.area, bins=20)
plt.title('Area Distribution')
plt.show()<jupyter_output><empty_output><jupyter_text>Create relevance function: small values for common cases and almost 1 for rare cases.<jupyter_code>import numpy as np
import math

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def relevance(x):
    x = np.array(x)
    return sigmoid(x - 50)
D = fires.copy()

target = 'area'

y_bar = D[target].median()

th = 0.999  #  threshold
y_bar<jupyter_output><empty_output><jupyter_text>Considering the rare cases<jupyter_code>rareL = D[(relevance(D[target]) > th) & (D[target] > y_bar)]  
rareL
rareH = D[(relevance(D[target]) > th) & (D[target] < y_bar)]  
rareH<jupyter_output><empty_output><jupyter_text>Build synthetic cases - step by step of Algorithm 2 in 'SMOTE for Regression'<jupyter_code>from sklearn.neighbors import KNeighborsRegressor

o = 200  # over sampling rate as percentage %
ng = 0//100

k = 3  # number of nearest neighbors to use for the generation

for index, case in D.iloc[:1].iterrows():
    knn = KNeighborsRegressor(n_neighbors = k+1)  # k+1 because the case is the nearest neighbor to itself
    knn.fit(D.drop(columns = [target]).values, D[[target]])  # warning: this step is very expensive
case
neighbors = knn.kneighbors(case.drop(labels = [target]).values.reshape(1, -1), return_distance=False).reshape(-1)
neighbors
index
neighbors = np.delete(neighbors, np.where(neighbors == index))
neighbors<jupyter_output><empty_output><jupyter_text>choose one of the neighbor randomly and create a new data point whose values are stored in attr<jupyter_code>x = D.iloc[neighbors[np.random.randint(k)]]
categorical_col = ['month_apr', 'month_aug', 'month_dec', 'month_feb', 'month_jan', 'month_jul',
                   'month_jun', 'month_mar', 'month_may', 'month_nov', 'month_oct', 'month_sep', 
                   'day_fri', 'day_mon', 'day_sat', 'day_sun', 'day_thu', 'day_tue', 'day_wed']
attr = {}

for a in D.columns:
    if a == target:  # skip
        continue
    if a in categorical_col:  # randomly pick one from either 'case' or 'x'
        if np.random.randint(2) == 0:
            attr[a] = case[a]
        else:
            attr[a] = x[a]
    else: 
        diff = case[a] - x[a]
        attr[a] = case[a] + np.random.randn() * diff  # there might be a mistake on the notebook, we follow the notion in the paper
        
from sklearn.metrics.pairwise import cosine_similarity

new = np.array(list(attr.values()))

# you can freely choose the distance function you like.

d1 = cosine_similarity(new.reshape(1, -1), case.drop(labels=[target]).values.reshape(1, -1))[0][0]
d2 = cosine_similarity(new.reshape(1, -1), x.drop(labels=[target]).values.reshape(1, -1))[0][0]

attr[target] = (d2 * case[target] + d1 * x[target]) / (d1 + d2)
attr[target]<jupyter_output><empty_output><jupyter_text>Let us put things together<jupyter_code>def get_synth_cases(D, target, o=200, k=3, categorical_col = []):
    '''
    Function to generate the new cases.
    INPUT:
        D - pd.DataFrame with the initial data
        target - string name of the target column in the dataset
        o - oversampling rate
        k - number of nearest neighbors to use for the generation
        categorical_col - list of categorical column names
    OUTPUT:
        new_cases - pd.DataFrame containing new generated cases
    '''
    new_cases = pd.DataFrame(columns = D.columns, dtype=object) # initialize the list of new cases 
    ng = o // 100 # the number of new cases to generate
    for index, case in D.iterrows():
        # find k nearest neighbors of the case
        knn = KNeighborsRegressor(n_neighbors = k+1) # k+1 because the case is the nearest neighbor to itself
        knn.fit(D.drop(columns = [target]).values, D[[target]])
        neighbors = knn.kneighbors(case.drop(labels = [target]).values.reshape(1, -1), return_distance=False).reshape(-1)
        neighbors = np.delete(neighbors, np.where(neighbors == index))
        for i in range(0, ng):
            # randomly choose one of the neighbors
            x = D.iloc[neighbors[np.random.randint(k)]]
            attr = {}          
            for a in D.columns:
                # skip target column
                if a == target:
                    continue;
                if a in categorical_col:
                    # if categorical then choose randomly one of values
                    if np.random.randint(2) == 0:
                        attr[a] = case[a]
                    else:
                        attr[a] = x[a]
                else:
                    # if continious column
                    diff = case[a] - x[a]
                    attr[a] = case[a] + np.random.randn() * diff
            # decide the target column
            new = np.array(list(attr.values()))
            d1 = cosine_similarity(new.reshape(1, -1), case.drop(labels = [target]).values.reshape(1, -1))[0][0]
            d2 = cosine_similarity(new.reshape(1, -1), x.drop(labels = [target]).values.reshape(1, -1))[0][0]
            attr[target] = (d2 * case[target] + d1 * x[target]) / (d1 + d2)
            
            # append the result
            new_cases = new_cases.append(attr,ignore_index = True)
                    
    return new_cases


def SmoteR(D, target, th = 0.999, o = 200, u = 100, k = 3, categorical_col = []):
    '''
    The implementation of SmoteR algorithm:
    https://core.ac.uk/download/pdf/29202178.pdf
    INPUT:
        D - pd.DataFrame - the initial dataset
        target - the name of the target column in the dataset
        th - relevance threshold
        o - oversampling rate
        u - undersampling rate
        k - the number of nearest neighbors
    OUTPUT:
        new_D - the resulting new dataset
    '''
    # median of the target variable
    y_bar = D[target].median()
    
    # find rare cases where target less than median
    rareL = D[(relevance(D[target]) > th) & (D[target] > y_bar)]  
    # generate rare cases for rareL
    new_casesL = get_synth_cases(rareL, target, o, k , categorical_col)
    
    # find rare cases where target greater than median
    rareH = D[(relevance(D[target]) > th) & (D[target] < y_bar)]
    # generate rare cases for rareH
    new_casesH = get_synth_cases(rareH, target, o, k , categorical_col)
    
    new_cases = pd.concat([new_casesL, new_casesH], axis=0)
    
    # undersample norm cases
    norm_cases = D[relevance(D[target]) <= th]
    # get the number of norm cases
    nr_norm = int(len(norm_cases) * u / 100)
    
    norm_cases = norm_cases.sample(min(len(D[relevance(D[target]) <= th]), nr_norm))
    
    # get the resulting dataset
    new_D = pd.concat([new_cases, norm_cases], axis=0)
    
    return new_D
Xs = SmoteR(D, target='area', th = 0.999, o = 300, u = 100, k = 10, categorical_col = ['month_apr',
       'month_aug', 'month_dec', 'month_feb', 'month_jan', 'month_jul',
       'month_jun', 'month_mar', 'month_may', 'month_nov', 'month_oct',
       'month_sep', 'day_fri', 'day_mon', 'day_sat', 'day_sun', 'day_thu',
       'day_tue', 'day_wed'])
Xs<jupyter_output><empty_output><jupyter_text>Now you know how it works. Let us use open source package directlyBe careful about the license, it is better to focus on MIT, Apache 2.0, and BSD III which can be applied on commercial software.<jupyter_code># conda install -c conda-forge imbalanced-learn -y
from imblearn.over_sampling import SMOTE
fires
fires['label'] = (relevance(D[target]) > th) & (D[target] > y_bar)
fires['label'] = fires['label'].astype(int)
fires.head(5)
fires[fires['label']==1]
from imblearn.over_sampling import SMOTENC

smote = SMOTENC(random_state=42, categorical_features=[np.where(fires.columns==cat)[0][0] for cat in categorical_col if len(np.where(fires.columns==cat)[0]) > 0])
X_resampled, y_resampled = smote.fit_resample(fires.drop(labels=['label'], axis=1), fires['label'])
X_resampled<jupyter_output><empty_output><jupyter_text>Image Regression<jupyter_code>import os
import random
from typing import Optional, Tuple, List
from glob import glob

import pandas as pd
import numpy as np
from PIL import Image
from tensorflow.keras.utils import Sequence
from tensorflow.keras.layers import Conv2D, BatchNormalization, Activation, add, Input, MaxPool2D, \
    GlobalAvgPool2D, Dropout, Dense
from tensorflow.keras.models import Model
from tensorflow.keras import regularizers
from tensorflow.keras import backend as K

# keras is now part of tensorflow API

def get_project_dir():

    """
    Get the full path to the repository
    """

#     dir_as_list = os.path.dirname(__file__).split("/")
#     index = dir_as_list.index("test")
#     project_directory = f"/{os.path.join(*dir_as_list[:index + 1])}"
    
    project_directory = "/Users/mling/OutSource/test"
    
    return project_directory
get_project_dir()
!pwd
"/Users/mling/論文數據"
class DataGenerator(Sequence):

    def __init__(self, dir: str, sample_size: Optional[int] = None, batch_size: int = 32, shuffle: bool = True):
        
        """
        Args:
            dir: directory in which images are stored
            sample_size: Optional; number of images will be sampled in each of sub_directory,
            from tying import Union
            sample_size: Union[int, None] -> Optional[int]
            if not provided all images in the dir are taken into account.
            batch_size: number of images in each of batch
            shuffle: if shuffle the order of the data
        """
        
        self.dir = dir
        self.batch_size = batch_size
        self.shuffle = shuffle
        self.sample_size = sample_size

        self.on_epoch_end()

        self.max = self.__len__()
        self.n = 0

    def __transform_to_dataframe(self) -> pd.DataFrame:
        
        """
        transform the data into a pandas dataframe to track the image files and the corresponding labels
        """
        
        dirs = glob(f"{get_project_dir()}/data/{self.dir}/*")
        #
        #dirs = glob(f"/Users/mling/論文數據/{self.dir}/*")
        data = []

        for dir in dirs:

            files = glob(f"{dir}/*.png")
            if self.sample_size:
                sampled_files = random.sample(files, min(self.sample_size, len(files)))
            else:
                sampled_files = files

            label = int(dir.split("/")[-1])

            for f in sampled_files:
                data.append([f, label])

        df = pd.DataFrame(data=data, columns=['filepath', 'label'], dtype=object)

        return df

    def on_epoch_end(self):
        
        self.df = self.__transform_to_dataframe()
        self.indices = self.df.index.tolist()

        self.index = np.arange(len(self.indices))
        if self.shuffle:
            np.random.shuffle(self.index)

    def __len__(self):
        #  Denotes the number of batches per epoch
        return int(np.ceil(len(self.indices) / self.batch_size))

    def __getitem__(self, index) -> Tuple[np.ndarray, np.ndarray]:
        # Generate one batch of data
        # Generate indices of the batch
        index = self.index[index * self.batch_size:(index + 1) * self.batch_size]
        # Find list of IDs
        batch = [self.indices[k] for k in index]
        # Generate data
        X, y = self.__get_data(batch)

        return X, y

    def __get_data(self, batch: List) -> Tuple[np.ndarray, np.ndarray]:

        df_batch = self.df.loc[batch]

        image_dataset = []
        labels = []

        for _, row in df_batch.iterrows():
            f = row['filepath']
            image_dataset.append(np.array(Image.open(f)) / 255.0)
            labels.append(row['label'])

        return np.array(image_dataset), np.array(labels)

    def __next__(self):
        
        """
        generate data of size batch_size
        """
        
        if self.n >= self.max:
            self.n = 0

        result = self.__getitem__(self.n)
        self.n += 1
        return result<jupyter_output><empty_output><jupyter_text>resnet10 model<jupyter_code>def conv2d_bn(x, nb_filter, kernel_size, strides=(1, 1), padding='same'):

    x = Conv2D(nb_filter, kernel_size=kernel_size, strides=strides, padding=padding,
               kernel_regularizer=regularizers.l2(0.00012))(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    return x


def conv2d_bn1(x, nb_filter, kernel_size, strides=(1, 1), padding='same'):
    x = Conv2D(nb_filter, kernel_size=kernel_size,
               strides=strides,
               padding=padding,
               kernel_regularizer=regularizers.l2(0.0001))(x)
    x = BatchNormalization()(x)
    return x


def shortcut(input, residual):
    input_shape = K.int_shape(input)
    residual_shape = K.int_shape(residual)
    stride_height = int(round(input_shape[1] / residual_shape[1]))
    stride_width = int(round(input_shape[2] / residual_shape[2]))
    equal_channels = input_shape[3] == residual_shape[3]

    identity = input
    # 如果維度不同，則使用1x1卷積進行調整
    if stride_width > 1 or stride_height > 1 or not equal_channels:
        identity = Conv2D(filters=residual_shape[3],
                          kernel_size=(1, 1),
                          strides=(stride_width, stride_height),
                          padding="valid",
                          kernel_regularizer=regularizers.l2(0.0001))(input)

    return add([identity, residual])


def shortcut1(input, residual,nb_filter):

    input = conv2d_bn1(input, nb_filter, kernel_size=(3, 3), strides=(2,2))
    input_shape = K.int_shape(input)
    residual_shape = K.int_shape(residual)
    stride_height = int(round(input_shape[1] / residual_shape[1]))
    stride_width = int(round(input_shape[2] / residual_shape[2]))
    equal_channels = input_shape[3] == residual_shape[3]

    identity = input
    # 如果維度不同，則使用1x1卷積進行調整
    if stride_width > 1 or stride_height > 1 or not equal_channels:
        identity = Conv2D(filters=residual_shape[3], kernel_size=(1, 1), strides=(stride_width, stride_height),
                          padding="valid", kernel_regularizer=regularizers.l2(0.0001))(input)

    return add([identity, residual])


def basic_block(nb_filter, strides=(1, 1)):

    def f(input):

        conv1 = conv2d_bn(input, nb_filter, kernel_size=(3, 3), strides=strides)
        residual = conv2d_bn1(conv1, nb_filter, kernel_size=(3, 3))

        return shortcut(input, residual)

    return f


def basic_block1(nb_filter, strides=(1, 1)):

    def f(input):

        conv1 = conv2d_bn1(input, nb_filter, kernel_size=(3, 3), strides=strides)

        return shortcut(input, conv1)

    return f


def resnet_10(input_shape=(64, 64, 3), nclass=1):
    input_ = Input(shape=input_shape)

    conv1 = conv2d_bn(input_, 64, kernel_size=(3, 3), strides=(2, 2))
    pool1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv1)

    conv3 = basic_block(64, strides=(1, 1))(pool1)
    conv3 = Activation('relu')(conv3)

    conv5 = basic_block1(128, strides=(2, 2))(conv3)
    conv5 = Activation('relu')(conv5)

    conv7 = basic_block1(256, strides=(2, 2))(conv5)
    conv7 = Activation('relu')(conv7)

    conv9 = basic_block1(512, strides=(2, 2))(conv7)
    conv9 = Activation('relu')(conv9)

    pool2 = GlobalAvgPool2D()(conv9)
    x = Dropout(0.4)(pool2)
    output_ = Dense(nclass, activation='linear')(x)

    model = Model(inputs=[input_], outputs=output_)
    model.summary()

    return model<jupyter_output><empty_output><jupyter_text>training process<jupyter_code>batch_size=10

model = resnet_10()
model.compile(optimizer='rmsprop', loss='mean_squared_error', metrics=['mse'])

train_generator = DataGenerator('train', sample_size=10, batch_size=batch_size)
val_generator = DataGenerator('test', shuffle=False, batch_size=batch_size)

# images, labels = next(train_generator)

model.fit_generator(generator=train_generator, steps_per_epoch=int(np.ceil(len(train_generator.df))/batch_size),
                    validation_data=val_generator, epochs=10)<jupyter_output>Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 64, 64, 3)]  0                                            
__________________________________________________________________________________________________
conv2d (Conv2D)                 (None, 32, 32, 64)   1792        input_1[0][0]                    
__________________________________________________________________________________________________
batch_normalization (BatchNorma (None, 32, 32, 64)   256         conv2d[0][0]                     
__________________________________________________________________________________________________
activation (Activation)         (None, 32, 32, 64)   0           batch_normalization[0][0]    [...]<jupyter_text>image augmentationif you are interested in the details: https://github.com/aleju/imgaug<jupyter_code>!conda install -c conda-forge imgaug -y
import imgaug as ia
import imgaug.augmenters as iaa

sometimes = lambda aug: iaa.Sometimes(1, aug)  # apply operations on 50% of input data

# what kind of image augmentations are properly depend on DOMAIN KNOWLEDGE

seq = iaa.Sequential([sometimes(iaa.Affine(
                    scale={'x': (0.95, 1.05), 'y': (0.95, 1.05)},
                    rotate=(-45, 45),
                    cval=0))])
images, labels = next(train_generator)
images.shape
labels
image_test = images[0]
ia.imshow(image_test)
ia.imshow(seq.augment_image(image_test))<jupyter_output><empty_output><jupyter_text>Let us add image augmentation component into image generator<jupyter_code>from imgaug.augmenters import Sequential

class DataGenerator(Sequence):

    def __init__(self, dir: str, sample_size: Optional[int] = None, batch_size: int = 32, shuffle: bool = True, 
                 image_augmentation: Optional[Sequential] = None):
        
        """
        Args:
            dir: directory in which images are stored
            sample_size: Optional; number of images will be sampled in each of sub_directory,
            if not provided all images in the dir are taken into account.
            batch_size: number of images in each of batch
            shuffle: if shuffle the order of the data
        """
        
        self.dir = dir
        self.batch_size = batch_size
        self.shuffle = shuffle
        self.sample_size = sample_size
        self.image_augmentation = image_augmentation  # new line

        self.on_epoch_end()

        self.max = self.__len__()
        self.n = 0

    def __transform_to_dataframe(self) -> pd.DataFrame:
        
        """
        transform the data into a pandas dataframe to track the image files and the corresponding labels
        """
        
        dirs = glob(f"{get_project_dir()}/data/{self.dir}/*")

        data = []
        
        # given df label: label
        # labels = np.unique(df[label])
        # for label in labels:
        #。 df_sub = self.df[self.df[label]==label]
        #。 files = df_sub['filename'].values
        
        
        for dir in dirs:

            files = glob(f"{dir}/*.png")
            if self.sample_size:  # modification
                if not self.image_augmentation:
                    sampled_files = random.sample(files, min(self.sample_size, len(files)))  # no repetition
                else:
                    sampled_files = random.choices(files, k=self.sample_size)  # repetition can take place
            else:
                sampled_files = files

            label = int(dir.split("/")[-1])

            for f in sampled_files:
                data.append([f, label])

        df = pd.DataFrame(data=data, columns=['filepath', 'label'], dtype=object)

        return df

    def on_epoch_end(self):
        
        self.df = self.__transform_to_dataframe()
        self.indices = self.df.index.tolist()

        self.index = np.arange(len(self.indices))
        if self.shuffle:
            np.random.shuffle(self.index)

    def __len__(self):
        #  Denotes the number of batches per epoch
        return int(np.ceil(len(self.indices) / self.batch_size))

    def __getitem__(self, index) -> Tuple[np.ndarray, np.ndarray]:
        # Generate one batch of data
        # Generate indices of the batch
        index = self.index[index * self.batch_size:(index + 1) * self.batch_size]
        # Find list of IDs
        batch = [self.indices[k] for k in index]
        # Generate data
        X, y = self.__get_data(batch)

        return X, y

    def __get_data(self, batch: List) -> Tuple[np.ndarray, np.ndarray]:

        df_batch = self.df.loc[batch]

        image_dataset = []
        labels = []
        
        for _, row in df_batch.iterrows():
            f = row['filepath']
            if not self.image_augmentation:
                image_dataset.append(np.array(Image.open(f)) / 255.0)
            else:
                image_dataset.append(self.image_augmentation.augment_image(np.array(Image.open(f))) / 255.0)
            labels.append(row['label'])

        return np.array(image_dataset), np.array(labels)

    def __next__(self):
        
        """
        generate data of size batch_size
        """
        
        if self.n >= self.max:
            self.n = 0

        result = self.__getitem__(self.n)
        self.n += 1
        return result
sometimes = lambda aug: iaa.Sometimes(1, aug)  # apply operations on 100% of input data

# what kind of image augmentations are properly depend on DOMAIN KNOWLEDGE

seq = iaa.Sequential([sometimes(iaa.Affine(
                    scale={'x': (0.95, 1.05), 'y': (0.95, 1.05)},
                    rotate=(-45, 45),
                    cval=0))])

train_generator = DataGenerator('train', sample_size=10, batch_size=batch_size, image_augmentation=seq)
val_generator = DataGenerator('test', shuffle=False, batch_size=batch_size)

# images, labels = next(train_generator)

model.fit_generator(generator=train_generator, steps_per_epoch=int(np.ceil(len(train_generator.df))/batch_size),
                    validation_data=val_generator, epochs=10)<jupyter_output>Epoch 1/10
 2/31 [>.............................] - ETA: 1s - loss: 65.0199 - mse: 64.8758
{'fraction_non_alphanumeric': 0.121684464347227, 'fraction_numerical': 0.018644505683775403, 'mean_word_length': 2.786273231170525, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 78, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '449890', 'license': 'MIT', 'n_tokens_mistral': 7767, 'n_tokens_neox': 7267, 'n_words': 1779, 'path': 'notebook/2021-08-07.ipynb', 'repo_name': 'mengchiehling/deepLearningTutorial', 'size': '23224'}