<jupyter_start><jupyter_text>Clean DataClean text dataset and obtain train, valid & test splitDataset: https://downloads.tatoeba.org/exports/<jupyter_code>import pandas as pd
import numpy as np
data_original = pd.read_csv('data/sentences.csv',sep='\t', encoding='utf8', header = None)
data_original.drop([0], axis=1,inplace=True)
data_original.columns = ['lang','sent']
data_original = data_original[data_original.sent.str.len()<=200]#Only select sentences less that 200 characters
data_original = data_original[data_original.sent.str.len()>=20]#Only select sentences greater that 20 characters
print(len(data_original)) #5788767 rows
print(data_original['lang'].nunique()) #314 unique languages
data_original.head()
#We want English, German, Russian, Spanish, French, Japanese, Portuguese, Italian
lang = ['eng','deu','spa','fra','por','ita']
data = data_original[data_original['lang'].isin(lang)]
print(len(data)) #2759972 rows
data.head()
data['lang'].value_counts()
#Select 50000 rows for each language
data_trim = pd.DataFrame(columns=['lang','sent'])
data_trim
for l in lang:
    data_trim = data_trim.append(data[data['lang'] ==l].sample(50000,random_state = 100))
data_trim['lang'].value_counts()
#Divide data into training, validation and testing
from sklearn.model_selection import train_test_split

X = data_trim['sent']
y = data_trim['lang']
X_train, X, y_train, y = train_test_split(X, y, test_size=0.30, random_state=101)
X_valid, X_test, y_valid, y_test = train_test_split(X, y, test_size=1/3, random_state=101)

#save datsets
train = pd.concat([pd.Series(y_train),pd.Series(X_train) ], axis=1)
valid = pd.concat([pd.Series(y_valid),pd.Series(X_valid) ], axis=1)
test = pd.concat([pd.Series(y_test),pd.Series(X_test) ], axis=1)
print(len(train), len(valid), len(test))

train.to_csv('data/train.csv')
valid.to_csv('data/valid.csv')
test.to_csv('data/test.csv')<jupyter_output>210000 60000 30000
{'fraction_non_alphanumeric': 0.1571503404924044, 'fraction_numerical': 0.03823991618648507, 'mean_word_length': 7.925233644859813, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '531289', 'license': 'MIT', 'n_tokens_mistral': 711, 'n_tokens_neox': 661, 'n_words': 138, 'path': 'src/1_clean_data.ipynb', 'repo_name': 'conorosully/language-identification', 'size': '1909'}