<jupyter_start><jupyter_text>Process Data<jupyter_code># Configurations
NUM_WORKERS = 4
NUM_CLASSES = 4
BATCH_SIZE = 64
NUM_EPOCHS = 120
LEARNING_RATE = 0.0001
RANDOM_SEED = 123
LOG_STEP = 150

damage_intensity_encoding = defaultdict(lambda: 0)
damage_intensity_encoding['destroyed'] = 3
damage_intensity_encoding['major-damage'] = 2
damage_intensity_encoding['minor-damage'] = 1
damage_intensity_encoding['no-damage'] = 0
def process_img(img_array, polygon_pts, scale_pct):
    height, width, _ = img_array.shape

    xcoords = polygon_pts[:, 0]
    ycoords = polygon_pts[:, 1]
    xmin, xmax = np.min(xcoords), np.max(xcoords)
    ymin, ymax = np.min(ycoords), np.max(ycoords)

    xdiff = xmax - xmin
    ydiff = ymax - ymin

    #Extend image by scale percentage
    xmin = max(int(xmin - (xdiff * scale_pct)), 0)
    xmax = min(int(xmax + (xdiff * scale_pct)), width)
    ymin = max(int(ymin - (ydiff * scale_pct)), 0)
    ymax = min(int(ymax + (ydiff * scale_pct)), height)

    return img_array[ymin:ymax, xmin:xmax, :]
def process_data(input_path, output_path, output_csv_path, val_split_pct):
    x_data = []
    y_data = []

    disasters = [folder for folder in os.listdir(input_path) if not folder.startswith('.')]
    disaster_paths = ([input_path + "/" +  d + "/images" for d in disasters])
    image_paths = []
    image_paths.extend([(disaster_path + "/" + pic) for pic in os.listdir(disaster_path)] for disaster_path in disaster_paths)
    img_paths = np.concatenate(image_paths)

    for img_path in tqdm(img_paths):

        img_obj = Image.open(img_path)
        img_array = np.array(img_obj)

        #Get corresponding label for the current image
        label_path = img_path.replace('png', 'json').replace('images', 'labels')
        label_file = open(label_path)
        label_data = json.load(label_file)

        for feat in label_data['features']['xy']:

            # only images post-disaster will have damage type
            try:
                damage_type = feat['properties']['subtype']
            except: # pre-disaster damage is default no-damage
                damage_type = "no-damage"
                continue

            poly_uuid = feat['properties']['uid'] + ".png"

            y_data.append(damage_intensity_encoding[damage_type])

            polygon_geom = shapely.wkt.loads(feat['wkt'])
            polygon_pts = np.array(list(polygon_geom.exterior.coords))
            poly_img = process_img(img_array, polygon_pts, 0.8)
            cv2.imwrite(output_path + "/" + poly_uuid, poly_img)
            x_data.append(poly_uuid)
    
    output_train_csv_path = os.path.join(output_csv_path, "train.csv")

    if(val_split_pct > 0):
        x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=val_split_pct)
        data_array_train = {'uuid': x_train, 'labels': y_train}
        data_array_test = {'uuid': x_test, 'labels': y_test}
        output_test_csv_path = os.path.join(output_csv_path, "test.csv")
        df_train = pd.DataFrame(data_array_train)
        df_test = pd.DataFrame(data_array_test)
        df_train.to_csv(output_train_csv_path)
        df_test.to_csv(output_test_csv_path)
    else: 
        data_array = {'uuid': x_data, 'labels': y_data}
        df = pd.DataFrame(data = data_array)
        df.to_csv(output_train_csv_path)
input_dir = r""
output_dir = r""
output_dir_csv = r""
val_split_pct = 0.0

process_data(input_dir, output_dir, output_dir_csv, float(val_split_pct))<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>def ordinal_loss(y_true, y_pred):
    weights = K.cast(K.abs(K.argmax(y_true, axis=1) - K.argmax(y_pred, axis=1))/(K.int_shape(y_pred)[1] - 1), dtype='float32')
    return (1.0 + weights) * keras.losses.categorical_crossentropy(y_true, y_pred )
def generate_xBD_baseline_model():
    weights = 'imagenet'
    inputs = Input(shape=(128, 128, 3))

    base_model = ResNet50(include_top=False, weights=weights, input_shape=(128, 128, 3))

    for layer in base_model.layers:
        layer.trainable = False

    x = Conv2D(32, (5, 5), strides=(1, 1), padding='same', activation='relu', input_shape=(128, 128, 3))(inputs)
    x = MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(x)

    x = Conv2D(64, (3, 3), strides=(1, 1), padding='same', activation='relu')(x)
    x = MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(x)

    x = Conv2D(64, (3, 3), strides=(1, 1), padding='same', activation='relu')(x)
    x = MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(x)

    x = Flatten()(x)

    base_resnet = base_model(inputs)
    base_resnet = Flatten()(base_resnet)

    concated_layers = Concatenate()([x, base_resnet])

    concated_layers = Dense(2024, activation='relu')(concated_layers)
    concated_layers = Dense(524, activation='relu')(concated_layers)
    concated_layers = Dense(124, activation='relu')(concated_layers)
    output = Dense(4, activation='relu')(concated_layers)

    model = Model(inputs=inputs, outputs=output)
    return model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13396004700352526, 'fraction_numerical': 0.020564042303172738, 'mean_word_length': 3.8499525166191835, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '448863', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1954, 'n_tokens_neox': 1837, 'n_words': 340, 'path': 'model/.ipynb_checkpoints/model-checkpoint.ipynb', 'repo_name': 'neal-macdonald/xView2_baseline', 'size': '5106'}