<jupyter_start><jupyter_text>Overlapped patch embedder<jupyter_code>EMBED_SIZE = 32
patch_embedder = PatchOverlapEmbeddings(input_channels=C, image_sizes=(H_reducted, W_reducted), stride=4, patch_size=7, embed_size=32)
number_of_patches = patch_embedder.number_of_patches
patch_height_resolution = patch_embedder.patch_height_resolution
patch_width_resolution = patch_embedder.patch_width_resoultion
patch_embedder_out, h_new, w_new = patch_embedder(images)
assert patch_embedder_out.shape == (N, H_reducted * W_reducted, EMBED_SIZE)<jupyter_output><empty_output><jupyter_text>Reduced Self Attention<jupyter_code>EMBED_SIZE = 32
NUM_HEADS = 2
REDUCTION = 2
DROPOUT = 0.0
attention = ReducedSelfAttention(NUM_HEADS, EMBED_SIZE, REDUCTION, DROPOUT)
attention_out = attention(patch_embedder_out, H_reducted, W_reducted)
assert attention_out.shape == (N, H_reducted * W_reducted, EMBED_SIZE)<jupyter_output><empty_output><jupyter_text>MIX FFN<jupyter_code>MLP_EXPANSION = 4
mix_ffn = MixFFN(EMBED_SIZE, MLP_EXPANSION, DROPOUT)
out_mix_ffn = mix_ffn(attention_out, H_reducted, W_reducted)
assert out_mix_ffn.shape == (N, H_reducted * W_reducted, EMBED_SIZE)<jupyter_output><empty_output><jupyter_text>Segformer Block<jupyter_code>segformer_block = SegformerBlock(NUM_HEADS, EMBED_SIZE, REDUCTION, MLP_EXPANSION, DROPOUT)
out_segformer_block = segformer_block(patch_embedder_out, H_reducted, W_reducted)
assert out_segformer_block.shape == (N, H_reducted * W_reducted, EMBED_SIZE)<jupyter_output><empty_output><jupyter_text>Segformer Encoder<jupyter_code>segformer_encoder = mitb0((H, W))
hidden_states = segformer_encoder(images)
for indx, hidden_state in enumerate(hidden_states):
    print(f"Hidden state {indx} shape: {hidden_state.shape}")<jupyter_output>Hidden state 0 shape: torch.Size([128, 32, 56, 56])
Hidden state 1 shape: torch.Size([128, 64, 28, 28])
Hidden state 2 shape: torch.Size([128, 160, 14, 14])
Hidden state 3 shape: torch.Size([128, 256, 7, 7])<jupyter_text>Segformer Decoder<jupyter_code>DECODER_HIDDEN_STATE = 256
DROPOUT = 0.0
NUM_CLASSES = 1000

layer_config = segformer_encoder.layer_configuration
segformer_decoder = SegformerDecoder(layer_config, DECODER_HIDDEN_STATE, NUM_CLASSES, DROPOUT)
decoder_output = segformer_decoder(hidden_states)
assert decoder_output.shape == (N, NUM_CLASSES, H_reducted, W_reducted)<jupyter_output><empty_output><jupyter_text>Segformer<jupyter_code>segformer_full = segformer_b2((H, W), NUM_CLASSES)
segmentation_out = segformer_full(images)
assert segmentation_out.shape == (N, NUM_CLASSES, H_reducted, W_reducted)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09772989611388996, 'fraction_numerical': 0.024624855713736054, 'mean_word_length': 10.818181818181818, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '586500', 'license': 'MIT', 'n_tokens_mistral': 1077, 'n_tokens_neox': 998, 'n_words': 161, 'path': 'segformer.ipynb', 'repo_name': 'minemile/pytorch-models', 'size': '2599'}