<commit_before>import os.path
import pandas as pd
import pickle
from sklearn.model_selection import train_test_split
import tensorflow.contrib.keras as keras

# 1. Load data into memory.

PWD = '<PASSWORD>/toxic_comments'
df  = pd.read_csv("{}/train.csv".format(PWD), sep=',')

X  = df['comment_text']
Y1 = df['toxic']
Y2 = df['severe_toxic']
Y3 = df['obscene']
Y4 = df['threat']
Y5 = df['insult']
Y6 = df['identity_hate']

split    = train_test_split(X, Y1, Y2, Y3, Y4, Y5, Y6, test_size=0.2)
X_train  = split[0]
X_test   = split[1]
Y1_train = split[2]
Y1_test  = split[3]
Y2_train = split[4]
Y2_test  = split[5]
Y3_train = split[6]
Y3_test  = split[7]
Y4_train = split[8]
Y4_test  = split[9]
Y5_train = split[10]
Y5_test  = split[11]
Y6_train = split[12]
Y6_test  = split[13]

# 2. Preprocess text fields.

tokenizer = keras.preprocessing.text.Tokenizer(num_words=20000)
tokenizer.fit_on_texts(X_train)

with open("{}/tokenizer.pickle".format(PWD), 'wb') as f:
   pickle.dump(tokenizer, f)

sequences = tokenizer.texts_to_sequences(X_train)
X_train   = keras.preprocessing.sequence.pad_sequences(sequences, maxlen=300)

sequences = tokenizer.texts_to_sequences(X_test)
X_test    = keras.preprocessing.sequence.pad_sequences(sequences, maxlen=300)

# 3. Build model.

ins = keras.layers.Input(shape=(300,), dtype='int32')<commit_msg>Re-use pickled tokenizer if continuing training.<commit_after>import os.path
import pandas as pd
import pickle
from sklearn.model_selection import train_test_split
import tensorflow.contrib.keras as keras

# 1. Load data into memory.

PWD = '<PASSWORD>/toxic_comments'
df  = pd.read_csv("{}/train.csv".format(PWD), sep=',')

X  = df['comment_text']
Y1 = df['toxic']
Y2 = df['severe_toxic']
Y3 = df['obscene']
Y4 = df['threat']
Y5 = df['insult']
Y6 = df['identity_hate']

split    = train_test_split(X, Y1, Y2, Y3, Y4, Y5, Y6, test_size=0.2)
X_train  = split[0]
X_test   = split[1]
Y1_train = split[2]
Y1_test  = split[3]
Y2_train = split[4]
Y2_test  = split[5]
Y3_train = split[6]
Y3_test  = split[7]
Y4_train = split[8]
Y4_test  = split[9]
Y5_train = split[10]
Y5_test  = split[11]
Y6_train = split[12]
Y6_test  = split[13]

# 2. Preprocess text fields.

path = "{}/tokenizer.pickle".format(PWD)
if os.path.isfile(path):
  with open(path, 'rb') as f:
    tokenizer = pickle.load(f)
else:
  tokenizer = keras.preprocessing.text.Tokenizer(num_words=20000)
  tokenizer.fit_on_texts(X_train)
  with open(path, 'wb') as f:
     pickle.dump(tokenizer, f)

sequences = tokenizer.texts_to_sequences(X_train)
X_train   = keras.preprocessing.sequence.pad_sequences(sequences, maxlen=300)

sequences = tokenizer.texts_to_sequences(X_test)
X_test    = keras.preprocessing.sequence.pad_sequences(sequences, maxlen=300)

# 3. Build model.

ins = keras.layers.Input(shape=(300,), dtype='int32')
{'fraction_non_alphanumeric': 0.1453447050461976, 'fraction_numerical': 0.04477611940298507, 'mean_word_length': 5.2555555555555555, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '651027', 'n_tokens_mistral': 1208, 'n_tokens_neox': 1145, 'n_words': 230}