<jupyter_start><jupyter_text>Test Pytorch functions<jupyter_code>import torch
import torch.nn as nn
exp_tensor = torch.rand([10, 60, 60, 3])
from pytorch_code.self_atten_utils import shape_list
shape_list(exp_tensor)<jupyter_output><empty_output><jupyter_text>split_heads_2d<jupyter_code>from pytorch_code.self_atten_utils import split_heads_2d<jupyter_output><empty_output><jupyter_text>split heads occurs after conv stem so lets conv first<jupyter_code># preproc needs same padding
preproc_layer = nn.Conv2d(3,16, kernel_size=7, stride=1, padding=3, dilation=1)
# stem
conv1 = nn.Conv2d(3, 16, kernel_size=7, stride=1, padding=3,
                               bias=False)
relu1 = nn.ReLU(inplace=True)
maxpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
# permute test <1
exp_tensor.shape
# permute test <2 - permute to channel first
exp_tensor.permute(0,3,1,2).shape
reshape = exp_tensor.permute(0,3,1,2)
processed = preproc_layer(reshape)
processed.shape
#filtered_batch = maxpool1(relu1(conv1(reshape_exp)))
#filtered_batch.shape
split = split_heads_2d(processed, 4)
split.shape
split = split_heads_2d(processed, 2)
split.shape<jupyter_output><empty_output><jupyter_text>combine_heads_2d<jupyter_code>from pytorch_code.self_atten_utils import combine_heads_2d
remerge = combine_heads_2d(split)
remerge.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10288675055514433, 'fraction_numerical': 0.03774981495188749, 'mean_word_length': 7.836601307189542, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '614771', 'license': 'MIT', 'n_tokens_mistral': 537, 'n_tokens_neox': 516, 'n_words': 88, 'path': 'Test_pytorch_code.ipynb', 'repo_name': 'Data-drone/attention_augmented_cnn', 'size': '1351'}