<commit_before>###############################################################################
#																			  #
# 	This program exhibits machine learning by reading in various pieces of 	  #
#	data by state and year, and based on correct examples, predicts the 	  #
#	outcome of the closest upcoming presidential election to that date in 	  #
#	that state.																  #
#																			  #
#	Used crime data because that was the data available to me.				  #
#																			  #
#		Accuracy: 0.581837381204 without energy data 						  #
#				  0.700831024931 with energy data							  #
# 		Accuracy values and significance of the difference between these	  #
#			values and a 50% accuracy (guessing)							  #
#			P value and statistical significance: 							  #
#  			The two-tailed P value is less than 0.0001						  #
#  			By conventional criteria, this difference is considered to be 	  #
#			extremely statistically significant. 							  #
#																			  #
#			 (source: http://graphpad.com/quickcalcs/oneSampleT2/)			  #
#																			  #
###############################################################################

import json
import sys
import math
import time

###############################################################################
#							 FUNCTION DEFINITIONS							  #
###############################################################################

# adjust the weights in response to an incorrect answer; alters the weights
def adjust_weights (data, weights, averages):
	for val in averages:
		if (data[val] > averages[val]):
			if (data['score'] > 0.0):
				weights[val] -= 1.0
	 		else:
	 			weights[val] += 1.0
	return

# set score which will be used to classify the year of the data; alters the data
def set_score (data):
	data['score'] = 0.0
	for val in weights:
	 	data['score'] += data[val] * weights[val]
	return data

#calculate averages of different values
def calculate_averages():
	averages = {
		"index":0.0,			  "violent":0.0, 		 "property":0.0,
		"murder":0.0,			  "forcible rape":0.0, 	 "robbery":0.0,
		"aggravated assault":0.0, "burglary":0.0, 		 "larceny theft":0.0,
		"vehicle theft":0.0, 	  "year":0.0, 			 "prev":0.0,
		"coal": 0.0, 			  "hydro": 0.0, 		 "natural gas": 0.0,
		"petroleum": 0.0, 		  "wind": 0.0, 			 "wood":	0.0,
		"nuclear": 0.0, 		  "biomass": 0.0,		 "other gas": 0.0,
		"geothermal": 0.0, 		  "pumped storage": 0.0, "solar": 0.0
	}
	for val in averages:
		count = 0.0
		score = 0.0
		for i in range (0, len(data)):
			score += data[i][val]
			count += 1.0
		averages[val] = score/count
	return averages

###############################################################################
#									MAIN									  #
###############################################################################

#open file containing data json
with open('data.json', 'r') as f:
    read_data = f.read()
    data = json.loads(read_data)
f.closed

wrong_results = []
num_correct_test = 0.0
count_test = 0.1
for x in range (0, len(data)):
	#adjust weights until reaching certain standard for accuracy
	#(stastistical significance or a time limit)
	num_correct = 0.0
	weights = {
		"index":0.0,			  "violent":0.0, 		 "property":0.0,
		"murder":0.0,			  "forcible rape":0.0, 	 "robbery":0.0,
		"aggravated assault":0.0, "burglary":0.0, 		 "larceny theft":0.0,
		"vehicle theft":0.0, 	  "year":0.0, 			 "prev":0.0,
		"coal": 0.0, 			  "hydro": 0.0, 		 "natural gas": 0.0,
		"petroleum": 0.0, 		  "wind": 0.0, 			 "wood":	0.0,
		"nuclear": 0.0, 		  "biomass": 0.0,		 "other gas": 0.0,
		"geothermal": 0.0, 		  "pumped storage": 0.0, "solar": 0.0
	}
	averages = calculate_averages()
	count = 0.1
	start = time.time()
	end = time.time()

	while ((num_correct/count < 0.95) and (end-start < .5)):
		for i in range (0, len(data)):
			if (i != x):
				data[i] = set_score(data[i])
				if ((data[i]['score']  > 0.0 and data[i]['result'] == 0) or
					(data[i]['score'] <= 0.0 and data[i]['result'] == 1)):
					adjust_weights(data[i], weights, averages)
				else:
					num_correct += 1.0
				count += 1.0
				# gets rid of the .1 at the end of count
				# which was originally there to avoid division by 0
				if (count == 1.1): 
					count = 1.0
		end = time.time()

	#now, decide for test data: set scores, measure accuracy of guesses
	data[x] = set_score(data[x])
	if ((data[x]['score']  > 0.0 and data[x]['result'] == 0) or
		(data[x]['score'] <= 0.0 and data[x]['result'] == 1)):
		wrong_results.append(data[x]['result'])
	else:
		num_correct_test += 1.0<commit_msg>made program more efficient<commit_after>###############################################################################
#																			  #
# 	This program exhibits machine learning by reading in various pieces of 	  #
#	data by state and year, and based on correct examples, predicts the 	  #
#	outcome of the closest upcoming presidential election to that date in 	  #
#	that state.																  #
#																			  #
#	Used crime data because that was the data available to me.				  #
#																			  #
#		Accuracy: 0.581837381204 without energy data 						  #
#				  0.716528162512 with energy data							  #
# 		Accuracy values and significance of the difference between these	  #
#			values and a 50% accuracy (guessing)							  #
#			P value and statistical significance: 							  #
#  			The two-tailed P value is less than 0.0001						  #
#  			By conventional criteria, this difference is considered to be 	  #
#			extremely statistically significant. 							  #
#																			  #
#			 (source: http://graphpad.com/quickcalcs/oneSampleT2/)			  #
#																			  #
###############################################################################

import json
import sys
import math
import time

###############################################################################
#							 FUNCTION DEFINITIONS							  #
###############################################################################

# adjust the weights in response to an incorrect answer; alters the weights
def adjust_weights (data, weights, averages):
	for val in averages:
		if (data[val] > averages[val]):
			if (data['score'] > 0.0):
				weights[val] -= 1.0
	 		else:
	 			weights[val] += 1.0
	return

# set score which will be used to classify the year of the data; alters the data
def set_score (data):
	data['score'] = 0.0
	for val in weights:
	 	data['score'] += data[val] * weights[val]
	return data

#calculate averages of different values
def calculate_averages():
	averages = {
		"index":0.0,			  "violent":0.0, 		 "property":0.0,
		"murder":0.0,			  "forcible rape":0.0, 	 "robbery":0.0,
		"aggravated assault":0.0, "burglary":0.0, 		 "larceny theft":0.0,
		"vehicle theft":0.0, 	  "year":0.0, 			 "prev":0.0,
		"coal": 0.0, 			  "hydro": 0.0, 		 "natural gas": 0.0,
		"petroleum": 0.0, 		  "wind": 0.0, 			 "wood":	0.0,
		"nuclear": 0.0, 		  "biomass": 0.0,		 "other gas": 0.0,
		"geothermal": 0.0, 		  "pumped storage": 0.0, "solar": 0.0
	}
	for val in averages:
		count = 0.0
		score = 0.0
		for i in range (0, len(data)):
			score += data[i][val]
			count += 1.0
		averages[val] = score/count
	return averages

###############################################################################
#									MAIN									  #
###############################################################################

#open file containing data json
with open('data.json', 'r') as f:
    read_data = f.read()
    data = json.loads(read_data)
f.closed

num_correct_test = 0.0
count_test = 0.1
for x in range (0, len(data)):
	#adjust weights until reaching certain standard for accuracy
	#(stastistical significance or a time limit)
	weights = {
		"index":0.0,			  "violent":0.0, 		 "property":0.0,
		"murder":0.0,			  "forcible rape":0.0, 	 "robbery":0.0,
		"aggravated assault":0.0, "burglary":0.0, 		 "larceny theft":0.0,
		"vehicle theft":0.0, 	  "year":0.0, 			 "prev":0.0,
		"coal": 0.0, 			  "hydro": 0.0, 		 "natural gas": 0.0,
		"petroleum": 0.0, 		  "wind": 0.0, 			 "wood":	0.0,
		"nuclear": 0.0, 		  "biomass": 0.0,		 "other gas": 0.0,
		"geothermal": 0.0, 		  "pumped storage": 0.0, "solar": 0.0
	}
	averages = calculate_averages()
	start = time.time()
	end = time.time()

	while (end-start < .5): # only give each data item .5 seconds
	#(otherwise it takes way too long)
		for i in range (0, len(data)):
			if (i != x):
				data[i] = set_score(data[i])
				if ((data[i]['score']  > 0.0 and data[i]['result'] == 0) or
					(data[i]['score'] <= 0.0 and data[i]['result'] == 1)):
					adjust_weights(data[i], weights, averages)
		end = time.time()

	#now, decide for test data: set scores, measure accuracy of guesses
	data[x] = set_score(data[x])
	if (not((data[x]['score']  > 0.0 and data[x]['result'] == 0) or
			(data[x]['score'] <= 0.0 and data[x]['result'] == 1))):
		num_correct_test += 1.0
{'fraction_non_alphanumeric': 0.24220162748643762, 'fraction_numerical': 0.03955696202531646, 'mean_word_length': 2.9986443741527338, 'pattern_counts': {'":': 96, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6516370', 'n_tokens_mistral': 3800, 'n_tokens_neox': 3011, 'n_words': 840}