<commit_before>import numpy as np
from sklearn import svm

def load_data(train_fn, test_fn):
	print "Loading data..."
	content = np.loadtxt(open(train_fn, 'rb'), delimiter=",", skiprows=1)
	data = content[:,1:]
	label = content[:,0]
	content = np.loadtxt(open(test_fn, 'rb'), delimiter=",", skiprows=1)
	test = content

	return data, label, test

def write_data(fn, pred):
	print "Writing predictions..."
	with open(fn, 'wb') as csvfile:
		writer = csv.writer(csvfile, delimiter=",")
		writer.writerow(["ImageId", "Label"])
		for i in range(len(pred)):
			writer.writerow([i+1, pred[i]])

def solve(data, label, test):
	print "Solving problem..."
	clf = svm.SVC()
	clf.fit(data, label)
	rst = clf.predict(test)

	return rst

if __name__ == "__main__":
	data, label, test = load_data('data/train.csv', 'data/test.csv')
	pred = solve(data, label, test)
	write_data('output.csv', pred)<commit_msg>POC working<commit_after>import numpy as np
from sklearn import svm
import matplotlib.pyplot as plt

def load_data(train_fn, test_fn):
	print "Loading data..."
	content = np.loadtxt(open(train_fn, 'rb'), delimiter=",", skiprows=1)
	data = content[:,1:]
	label = content[:,0]
	content = np.loadtxt(open(test_fn, 'rb'), delimiter=",", skiprows=1)
	test = content

	return data, label, test

def write_data(fn, pred):
	print "Writing predictions..."
	with open(fn, 'wb') as csvfile:
		writer = csv.writer(csvfile, delimiter=",")
		writer.writerow(["ImageId", "Label"])
		for i in range(len(pred)):
			writer.writerow([i+1, pred[i]])

def preprocess(data, label, test, lite=False):
	print "Preprocessing data..."

	# Scale
	data = data / 255 * 2 - 1
	test = test / 255 * 2 - 1
	label = label.astype(int)

	# Shuffle
	shuffle = np.random.permutation(len(data))
	data = data[shuffle,]
	label = label[shuffle,]

	# Lite set
	if lite:
		data = data[1:10000, :]
		test = test[1:10000, :]
		label = label[1:10000]

	return data, label, test

def split_valid(data, label, fold):
	split = int(data.shape[0] / fold)

	data_t = data[0:split,]
	label_t = label[0:split,]
	data_v = data[split:,]
	label_v = label[split:,]

	return data_t, label_t, data_v, label_v

def solve(data, label):
	print "Solving problem..."
	clf = svm.SVC()
	clf.fit(data, label)

	return clf

def evaluate(clf, data, label):
	print "Evaluating the classifier..."

	pred = clf.predict(data)
	mse = ((pred - label) ** 2).mean()

	print "MSE is ", mse

	return mse

if __name__ == "__main__":
	LITE = True

	data, label, test = load_data('data/train.csv', 'data/test.csv')
	data, label, test = preprocess(data, label, test, LITE)

	# Validation
	data_t, label_t, data_v, label_v = split_valid(data, label, 5)
	clf = solve(data_t, label_t)
	rst = evaluate(clf, data_v, label_v)

	write_data('output.csv', pred)
{'fraction_non_alphanumeric': 0.1593597773138483, 'fraction_numerical': 0.014961725817675714, 'mean_word_length': 3.659643435980551, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7494689', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1066, 'n_words': 273}