<commit_before>import os
import numpy as np
from os.path import dirname, abspath

from src import Utils
Utils.set_keras_backend("tensorflow")

from keras.models import Sequential
from keras.layers import Dense


data_dir = os.path.join(dirname(dirname(dirname(dirname(abspath(__file__))))), 'data')
training_data_dir = os.path.join(data_dir, 'training_data')

X = []
Y = []

for csv_file in os.listdir(training_data_dir):
    if csv_file != 'game_100672.sgf.csv': continue  # dev restriction

    data = np.genfromtxt(os.path.join(training_data_dir, csv_file), dtype=float, delimiter=';')

    for line in data:
        x = line[:-2]
        y = [0 for _i in range(82)]  # pos 0 is PASS
        if line[81] is -1:  # = PASS
            y[0] = 1
        else:
            y[int(line[81]) + 1] = 1
        X.append(x)
        Y.append(y)

# it might be (much) better to append to a numpy array right away?
# np.concatenate would do the job, but seems very memory costly to reassign it in a loop?
# a = np.array([[1,2,3]]) b = np.array([[4,5,6]]) c = np.concatenate((a,b))
X = np.array(X)
Y = np.array(Y)

# set up network topology
model = Sequential()
# first arg of Dense is # of neurons
model.add(Dense(162, input_dim=81, activation='relu'))
model.add(Dense(82, activation='softmax'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(X, Y, epochs=1)
scores = model.evaluate(X, Y)
print("\n%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))

print(model.predict(X))<commit_msg>save trained model as .h5 file<commit_after>import os
import numpy as np
from os.path import dirname, abspath

from src import Utils
Utils.set_keras_backend("tensorflow")

from keras.models import Sequential
from keras.layers import Dense

project_dir = dirname(dirname(dirname(dirname(abspath(__file__)))))
training_data_dir = os.path.join(project_dir, 'data/training_data')

X = []
Y = []

for csv_file in os.listdir(training_data_dir):
    if csv_file != 'game_100672.sgf.csv': continue  # dev restriction

    data = np.genfromtxt(os.path.join(training_data_dir, csv_file), dtype=float, delimiter=';')

    for line in data:
        x = line[:-2]
        y = [0 for _i in range(82)]  # pos 0 is PASS, so we need 1+81
        move_idx = int(line[81])
        if move_idx is -1:  # = PASS
            y[0] = 1
        else:
            y[move_idx + 1] = 1
        X.append(x)
        Y.append(y)

# it might be (much) better to append to a numpy array right away?
# np.concatenate would do the job, but seems very memory costly to reassign it in a loop?
# a = np.array([[1,2,3]]) b = np.array([[4,5,6]]) c = np.concatenate((a,b))
X = np.array(X)
Y = np.array(Y)

# set up network topology
model = Sequential()
# first arg of Dense is # of neurons
model.add(Dense(162, input_dim=81, activation='relu'))
model.add(Dense(82, activation='softmax'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(X, Y, epochs=1)
scores = model.evaluate(X, Y)
print("\n%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))

model.save(os.path.join(project_dir, 'src/learn/dev_ben/model.h5'))
{'fraction_non_alphanumeric': 0.14853689567430026, 'fraction_numerical': 0.026399491094147583, 'mean_word_length': 3.8684210526315788, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '211270', 'n_tokens_mistral': 1199, 'n_tokens_neox': 1138, 'n_words': 306}