<jupyter_start><jupyter_text>**Problem**Split IMDB training dataset into two: 20K and 5K and create two .csv files accordingly.Before spliting, the training data must be shuffled to ensure class balance.<jupyter_code># Enable intellisense
%config IPCompleter.greedy=True
# Import modules
import pandas as pd
import numpy as np
import bert
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import  Model
from tensorflow.keras.layers import Input, Dense, Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard
from tqdm import tqdm
import matplotlib.pyplot as plt

print("TensorFlow Version:",tf.__version__)
print("Hub version: ",hub.__version__)
print("GPU is", "available" if tf.config.list_physical_devices('GPU') else "NOT AVAILABLE")
pd.set_option('display.max_colwidth',1000)
pd.options.display.max_rows = 10000
import sys
sys.path.append("../helpers")
import imdb_preprocess_functions as nist_imdb
dir(nist_imdb)
# Now load the Stanford IMDB training and test dataset.
[df_train, df_test] = nist_imdb.get_imdb_df_data('../data/imdb_master.csv')
# Shuffle the dataset
df = df_train.sample(frac=1, random_state=0)
df.shape
SPLIT_TRAIN_SIZE = 17500
df_train_train = df[:SPLIT_TRAIN_SIZE]
df_train_test = df[SPLIT_TRAIN_SIZE:]
df_train_train.shape
df_train_test.shape
# Check the target class balance
df_train_train[nist_imdb.label_column].value_counts()
# Check the target class balance
df_train_test[nist_imdb.label_column].value_counts()
file_train_train = 'imdb_train_split_' + str(SPLIT_TRAIN_SIZE) + '.csv'
file_train_test = 'imdb_train_split_'  + str(25000 - SPLIT_TRAIN_SIZE) + '.csv'
df_train_train.to_csv(file_train_train, index=False, columns = [nist_imdb.text_column, nist_imdb.label_column, nist_imdb.file_column])
df_train_test.to_csv(file_train_test, index=False, columns = [nist_imdb.text_column, nist_imdb.label_column, nist_imdb.file_column])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09215686274509804, 'fraction_numerical': 0.011764705882352941, 'mean_word_length': 7.399176954732511, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '5969', 'license': 'CNRI-Python', 'n_tokens_mistral': 733, 'n_tokens_neox': 713, 'n_words': 165, 'path': 'models/model_{3,4}/misc-src/split-imdb-train-dataset.ipynb', 'repo_name': 'generic-github-user/STVM_NLP_Research', 'size': '2040'}