<jupyter_start><jupyter_text>Train RNN to identify russian language Some configs that you can modify,more configs in config.yaml<jupyter_code>OUTPUT_DIR = "./"
TRAIN_PERCENT = 0.6
VALIDATION_PERCENT = 0.2
TEST_PERCENT = 0.2<jupyter_output><empty_output><jupyter_text>Cloning repo and extract data<jupyter_code>%%bash
# rm -rf ./Russian-Language-Identification
git clone https://github.com/sergeisoly/Russian-Language-Identification
cd ./Russian-Language-Identification
make
make data<jupyter_output><empty_output><jupyter_text>Split dataset to train-validation-test both WILI Dataset and Taiga (Lenta) Dataset<jupyter_code>%%bash -s "$TRAIN_PERCENT" "$VALIDATION_PERCENT"
cd ./Russian-Language-Identification
python3 src/split_datasets.py ./data/lenta/ ./data/ $1 $2<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>%cd ./Russian-Language-Identification
!python3 src/train.py --config ./config.yaml --output_dir {OUTPUT_DIR}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11582213029989659, 'fraction_numerical': 0.010341261633919338, 'mean_word_length': 7.0, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '107579', 'license': 'MIT', 'n_tokens_mistral': 353, 'n_tokens_neox': 341, 'n_words': 64, 'path': 'notebooks/train_in_colab.ipynb', 'repo_name': 'sergeisoly/Russian-Language-Identification', 'size': '967'}