<jupyter_start><jupyter_text>Testing the main scripts<jupyter_code>import datetime
import os
import sys
import torch

from torch.utils.data import DataLoader
from torchvision import transforms

sys.path.append('d:/gsn-projekt')

import amphibian.preprocess.preprocessing as preproc

from amphibian.fetch.reader import AmphibianReader
from amphibian.preprocess.train_test_split import TrainTestSplit
ar = AmphibianReader('../data/all_values/banking', datetime.datetime(2010,1,4), datetime.datetime(2018, 12, 31))
_ = ar.create_torch()
ar.torch['AMERICA'].shape
ar.torch['AMERICA'][2000:2342, :, :]
ar.torch['ASIA_PACIFIC'].shape<jupyter_output><empty_output><jupyter_text>Balancing classes<jupyter_code>tts = TrainTestSplit(ar, int_start=0, int_end=2342, input_reg='ASIA_PACIFIC', pred_reg='EMEIA')
eps = 0.01
tsds = preproc.TimeSeriesDataset(
    tts, int_len=10, 
    transform=transforms.Compose([
        preproc.FillNaN(), preproc.Normalizing(),
        preproc.DummyFillNaN(), preproc.Formatting(),
        preproc.FormattingY()
    ])
)
len(tsds)
dl = DataLoader(tsds, batch_size=len(tsds))
for _, batch in enumerate(dl):
    print(torch.sum(batch['train_y'] == 0))
    print(torch.sum(batch['train_y'] == 1))
    print(torch.sum(batch['train_y'] == 2))<jupyter_output>tensor(6959, device='cuda:0')
tensor(4587, device='cuda:0')
tensor(7094, device='cuda:0')<jupyter_text>The default threshold of 50bp can remain as is.<jupyter_code>run_path = r'data/run_attention_{:%Y%m%d_%H%M%S}.csv'.format(datetime.datetime.now())
run_path
os.makedirs(run_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14637223974763408, 'fraction_numerical': 0.032807570977917984, 'mean_word_length': 6.851485148514851, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '663445', 'license': 'MIT', 'n_tokens_mistral': 625, 'n_tokens_neox': 587, 'n_words': 90, 'path': 'tests/test_main.ipynb', 'repo_name': 'jkoscialkowski/gsn-projekt', 'size': '1585'}