<commit_before>
import grace
import grace.times
import grace.ols
import numpy as np
import matplotlib.pyplot as plt
import cPickle as pickle
import sklearn.mixture as skm
import sklearn.decomposition as skd
import time
import mpl_toolkits.basemap as maps
import matplotlib as mpl
import grace
import grace.times

days=grace.ols.time_vector()
number_of_pcs=10

colors_hex = [
	"a6cee3", "1f78b4", "b2df8a", "33a02c", "fb9a99", "e31a1c",
	"fdbf6f", "ff7f00", "cab2d6", "6a3d9a", "ffff99", "b15928","edf8b1"
	,"bcbddc", "636363", 
]
colors_rgb = map(lambda hex: [ord(c) for c in hex.decode('hex')], colors_hex)


def write_latex_table(x,v):
	print 'writing latex table'
	covs=np.zeros([341,x.shape[0]])
	print covs.shape
	for i,elem in enumerate(x):
		covs[:,i]=1.96*np.sqrt(np.dot(np.diag(elem).reshape(1,number_of_pcs), v.T[:number_of_pcs]))
	with open('HPC-output/covariance_structure.txt','wb') as f:
		f.write('\\begin{tabular}{l|c|c|c|c|c|c|c|c}')
		f.write('\n\tday')
		for i in range(8):
			f.write(' &\t cluster ' +str(i))
		f.write(' \\\\')
		for i,row in enumerate(covs):
			f.write('\n\t'+str(i+1))
			for elem in row:
				f.write('\t & ' + str(elem))
			f.write('\\\\')

		f.write('\n\\end{tabular}')
			
	print 'latex table written'
	

def plot_clusters(model,X,V):
	y=model.predict(X)
	y=y.reshape(180,360)[::-1]
	shape=y.shape
	#print y.shape
	fig = plt.figure(figsize=(12, 6))

	m = maps.Basemap(projection='cyl', lon_0=0, resolution='c')
	m.drawcoastlines(linewidth=2.5, color="white")
	m.drawparallels(np.arange(-90.,120.,30.), labels=[1,0,0,0])
	m.drawmeridians(np.arange(0.,420.,60.), labels=[0,0,0,1])
	
	centroids=model.means_
	covariance=model.covars_
	write_latex_table(covariance,v)
	cmap=plt.cm.jet
	cmaplist=[cmap(i) for i in range(cmap.N)]
	cmap=cmap.from_list('Custom cmap',cmaplist,cmap.N)
	bounds=np.linspace(0,len(centroids),len(centroids)+1)
	norm=mpl.colors.BoundaryNorm(bounds,cmap.N)
	
	world = np.ones((shape[0], shape[1], 3)).astype('uint8') * 255

	for i in range(0,shape[0]):
		for j in range(0,shape[1]):
			world[i,j,:]=colors_rgb[y[i,j]]
	
	im = m.imshow(world,cmap=cmap,norm=norm)
	plt.title('Clusters')
	
	fig = plt.figure(figsize=(12, 6))
	labels=model.predict(centroids)
#	print centroids,model.weights_,labels
	for i,series in enumerate(np.dot(centroids,v.T[:number_of_pcs])):
		plt.plot(days, series.ravel(),lw=2,c='#' + colors_hex[i],label='cluster '+str(i+1))
		#uncomment for plus/minus 2x sd ontop

		#plt.plot(days, (series.ravel()+np.dot(2*np.sqrt(np.diag(covariance[i])).reshape(1,number_of_pcs),v.T[:number_of_pcs])).ravel(),'--',lw=2,c=cmap(norm( [float(labels[i])] )).ravel())
		#plt.plot(days,(series.ravel() -np.dot(2*np.sqrt(np.diag(covariance[i])).reshape(1,number_of_pcs),v.T[:number_of_pcs])).ravel(),'--',lw=2,c=cmap(norm( [float(labels[i])] )).ravel())
	plt.legend(loc=3)
	plt.title('Cluster centroids')

	plt.show()
	return 0


def plot_pca_clusters(model,kpca,X):
	y=model.predict(X)
	y=y.reshape(180,360)[::-1]
	shape=y.shape
	#print y.shape
	fig = plt.figure(figsize=(12, 6))

	m = maps.Basemap(projection='cyl', lon_0=0, resolution='c')
	m.drawcoastlines(linewidth=2.5, color="white")
	m.drawparallels(np.arange(-90.,120.,30.), labels=[1,0,0,0])
	m.drawmeridians(np.arange(0.,420.,60.), labels=[0,0,0,1])
	
	centroids=model.means_
	covariance=model.covars_
	write_latex_table(covariance,v)
	cmap=plt.cm.jet
	cmaplist=[cmap(i) for i in range(cmap.N)]
	cmap=cmap.from_list('Custom cmap',cmaplist,cmap.N)
	bounds=np.linspace(0,len(centroids),len(centroids)+1)
	norm=mpl.colors.BoundaryNorm(bounds,cmap.N)
	
	world = np.ones((shape[0], shape[1], 3)).astype('uint8') * 255<commit_msg>Fixed errors in kpca clustering<commit_after>
import grace
import grace.times
import grace.ols
import numpy as np
import matplotlib.pyplot as plt
import cPickle as pickle
import sklearn.mixture as skm
import sklearn.decomposition as skd
import time
import mpl_toolkits.basemap as maps
import matplotlib as mpl
import grace
import grace.times

days=grace.ols.time_vector()
number_of_pcs=10

colors_hex = [
	"a6cee3", "1f78b4", "b2df8a", "33a02c", "fb9a99", "e31a1c",
	"fdbf6f", "ff7f00", "cab2d6", "6a3d9a", "ffff99", "b15928","edf8b1"
	,"bcbddc", "636363", 
]
colors_rgb = map(lambda hex: [ord(c) for c in hex.decode('hex')], colors_hex)


def write_latex_table(x,v):
	print 'writing latex table'
	covs=np.zeros([341,x.shape[0]])
	print covs.shape
	for i,elem in enumerate(x):
		covs[:,i]=1.96*np.sqrt(np.dot(np.diag(elem).reshape(1,number_of_pcs), v.T[:number_of_pcs]))
	with open('HPC-output/covariance_structure.txt','wb') as f:
		f.write('\\begin{tabular}{l|c|c|c|c|c|c|c|c}')
		f.write('\n\tday')
		for i in range(8):
			f.write(' &\t cluster ' +str(i))
		f.write(' \\\\')
		for i,row in enumerate(covs):
			f.write('\n\t'+str(i+1))
			for elem in row:
				f.write('\t & ' + str(elem))
			f.write('\\\\')

		f.write('\n\\end{tabular}')
			
	print 'latex table written'
	

def plot_pca_clusters(model,kpca,X):
	y=model.predict(X)
	y=y.reshape(180,360)[::-1]
	shape=y.shape
	#print y.shape
	fig = plt.figure(figsize=(12, 6))

	m = maps.Basemap(projection='cyl', lon_0=0, resolution='c')
	m.drawcoastlines(linewidth=2.5, color="white")
	m.drawparallels(np.arange(-90.,120.,30.), labels=[1,0,0,0])
	m.drawmeridians(np.arange(0.,420.,60.), labels=[0,0,0,1])
	
	centroids=model.means_
	covariance=model.covars_

	cmap=plt.cm.jet
	cmaplist=[cmap(i) for i in range(cmap.N)]
	cmap=cmap.from_list('Custom cmap',cmaplist,cmap.N)
	bounds=np.linspace(0,len(centroids),len(centroids)+1)
	norm=mpl.colors.BoundaryNorm(bounds,cmap.N)
	
	world = np.ones((shape[0], shape[1], 3)).astype('uint8') * 255
{'fraction_non_alphanumeric': 0.20099343622494234, 'fraction_numerical': 0.045769026077700906, 'mean_word_length': 5.540603248259861, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4592893', 'n_tokens_mistral': 2710, 'n_tokens_neox': 2506, 'n_words': 390}