<commit_before># Explosound
# 15th November 2014
# see LICENSE for more information

"Creates a gexf valid file containing all files found in the specified folder as nodes, and random\
edges between them"

import sys
import os
from os import listdir
from os.path import isfile, join
import fnmatch
import random
from pymir import AudioFile
import matplotlib.pyplot as plt
import numpy as np
import fnmatch
import pickle as pkl


#print 'Number of arguments:', len(sys.argv), 'arguments.'
#print 'Argument List:', str(sys.argv)
#print 'Folder =', str(sys.argv[1])

# Make the job list

# Creates a list of all wav files found in all folders
# recursively
def make_job_list(folderPath) :
	job_list = []
	for root, directories, filenames in os.walk(folderPath):
		for filename in filenames:
			path = os.path.join(root,filename)
			if fnmatch.fnmatch(filename, '*.wav'):
				job_list.append(str(path))

	return job_list

def get_gexf_header():
	return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
	<gexf xmlns=\"http://www.gexf.net/1.2draft\" version=\"1.2\" xmlns:viz=\"http://www.gexf.net/1.2draft/viz\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.gexf.net/1.2draft http://www.gexf.net/1.2draft/gexf.xsd\">\
	  <meta lastmodifieddate=\"2014-10-05\">\
	    <creator>Gephi 0.8.1</creator>\
	    <description></description>\
	  </meta>\
	  <graph defaultedgetype=\"undirected\" mode=\"static\">\
	    <attributes class=\"node\" mode=\"static\">\
	      <attribute id=\"modularity_class\" title=\"Modularity Class\" type=\"integer\"></attribute>\
	    </attributes>"

def get_gexf_footer():
	return "  </graph>\
</gexf>"

def get_gexf_node(id, filePath, size, x, y, z, r, g, b):
	return "<node id=\""+str(id)+ "\" label=\"" + str(filePath) + "\">\
        <attvalues>\
          <attvalue for=\"modularity_class\" value=\"0\"></attvalue>\
        </attvalues>\
        <viz:size value=\"" +str(size)+ "\"></viz:size>\
        <viz:position x=\""+str(x)+"\" y=\""+str(y)+"\" z=\""+str(z)+"\"></viz:position>\
        <viz:color r=\""+str(r)+"\" g=\""+str(g)+"\" b=\""+str(b)+"\"></viz:color>\
      </node>"

def get_gexf_edge(id, source, target, proximity):
	return "<edge id=\""+str(id)+"\" source=\""+str(source)+"\" target=\""+ str(target) + "\" weight=\""+ str(proximity) + "\"/>"

def save_to_file(filePath, gexfToSave):
	text_file = open(filePath, "w")
	text_file.write(gexfToSave)
	text_file.close()

def get_features(job_list, force_rescan):
	print "get_features"
	if(force_rescan == True):
		print ">force_rescan on " + str(len(job_list)) + " jobs"
		analysed = 0
		audiofile_features = {}
		for i in range(len(job_list)):
			try:
				print "> " + str(i) + " / " + str(len(job_list)),
				audiofile = AudioFile.open(job_list[i])
				print ",s",
				spectrum = audiofile.spectrum()				
				rms = audiofile.rms()
				print ",rms",
				centroid = spectrum.centroid
				print ",centroid",
				flatness = -1
				try:
					flatness = str(spectrum.flatness())
					pass
				except:
					pass
				finally:
					pass

				print ",flatness",				
				features = [rms, centroid, flatness]
				print ",done!"
				audiofile_features[job_list[i]] = features
				analysed = analysed + 1
			except:
				#print "Exception: " + job_list[i]
				pass
			else:
				pass
			finally:
				pass	

		print "<force_rescan " + str(analysed) + "/" + str(len(job_list)) + " scanned."

		rms = []
		centroid = []
		flatness = []

		for i in audiofile_features.keys():
			rms.append(audiofile_features[i][0])
			centroid.append(audiofile_features[i][1])
			flatness.append(float(audiofile_features[i][2]))

		#serialisation avec pickle 
		with open('./rms.pkl', 'wb') as f:
			pkl.dump(rms,f)
		with open('./centroid.pkl', 'wb') as f:
			pkl.dump(centroid,f)
		with open('./flatness.pkl', 'wb') as f:
			pkl.dump(flatness,f)
		with open('./features.pkl', 'wb') as f:
			pkl.dump(audiofile_features,f)			

	#ouvrir un serialized
	with open('./rms.pkl', 'rb') as f:
		rms = pkl.load(f)
	with open('./centroid.pkl', 'rb') as f:
		centroid = pkl.load(f)
	with open('./flatness.pkl', 'rb') as f:
		flatness = pkl.load(f)
	with open('./features.pkl', 'rb') as f:
		features = pkl.load(f)

	filtered_features = features #TODO
	return filtered_features

def get_similarity(features):
	# TODO generate similarity matrix
	return 0




#MAIN ======================================================================================
folderPath = sys.argv[1]
defaultSize = 10
defaultR = 153
defaultG = 153
defaultB = 153
minX = -500
maxX = 500
minY = -500
maxY = 500
minZ = -10
maxZ = 10
edgeCreationProbability = 0.0013
edgeDefaultProximity = 1

print "Hello =)"
print "Generating job list for folder " + folderPath
job_list = make_job_list(folderPath)
print "Generating similarity matrix"
features = get_features(job_list)
similarity = get_similarity(features)

print "____________________________"
print "Generating the resulting graph"
print ">nodes"
# Generate all nodes
node_list = []
id = 0
for job in make_job_list(folderPath):
	node_list.append(get_gexf_node(id, job, 10, random.uniform(minX, maxX), random.uniform(minY, maxY), random.uniform(minZ, maxZ),\
		defaultR, defaultG, defaultB))
	id += 1

print ">edges"
# Generate all edges
edge_list = []
id = 0
for nodeId1 in range(0, len(node_list)):
	for nodeId2 in range(0, len(node_list)):
		if(random.uniform(0, 1) < edgeCreationProbability):
			edge_list.append(get_gexf_edge(id, nodeId1, nodeId2, edgeDefaultProximity))
			id+=1

print ">gexf header and footer"
# Generate the gexf
gexf_file = get_gexf_header()
<commit_msg>Fixed serialization : need full path.<commit_after># Explosound
# 15th November 2014
# see LICENSE for more information

"Creates a gexf valid file containing all files found in the specified folder as nodes, and random\
edges between them"

import sys
import os
from os import listdir
from os.path import isfile, join
import fnmatch
import random
from pymir import AudioFile
import matplotlib.pyplot as plt
import numpy as np
import fnmatch
import pickle as pkl
from sklearn import preprocessing as ppr


#print 'Number of arguments:', len(sys.argv), 'arguments.'
#print 'Argument List:', str(sys.argv)
#print 'Folder =', str(sys.argv[1])

# Make the job list

# Creates a list of all wav files found in all folders
# recursively
def make_job_list(folderPath) :
	job_list = []
	for root, directories, filenames in os.walk(folderPath):
		for filename in filenames:
			path = os.path.join(root,filename)
			if fnmatch.fnmatch(filename, '*.wav'):
				job_list.append(str(path))

	return job_list

def get_gexf_header():
	return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
	<gexf xmlns=\"http://www.gexf.net/1.2draft\" version=\"1.2\" xmlns:viz=\"http://www.gexf.net/1.2draft/viz\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.gexf.net/1.2draft http://www.gexf.net/1.2draft/gexf.xsd\">\
	  <meta lastmodifieddate=\"2014-10-05\">\
	    <creator>Gephi 0.8.1</creator>\
	    <description></description>\
	  </meta>\
	  <graph defaultedgetype=\"undirected\" mode=\"static\">\
	    <attributes class=\"node\" mode=\"static\">\
	      <attribute id=\"modularity_class\" title=\"Modularity Class\" type=\"integer\"></attribute>\
	    </attributes>"

def get_gexf_footer():
	return "  </graph>\
</gexf>"

def get_gexf_node(id, filePath, size, x, y, z, r, g, b):
	return "<node id=\""+str(id)+ "\" label=\"" + str(filePath) + "\">\
        <attvalues>\
          <attvalue for=\"modularity_class\" value=\"0\"></attvalue>\
        </attvalues>\
        <viz:size value=\"" +str(size)+ "\"></viz:size>\
        <viz:position x=\""+str(x)+"\" y=\""+str(y)+"\" z=\""+str(z)+"\"></viz:position>\
        <viz:color r=\""+str(r)+"\" g=\""+str(g)+"\" b=\""+str(b)+"\"></viz:color>\
      </node>"

def get_gexf_edge(id, source, target, proximity):
	return "<edge id=\""+str(id)+"\" source=\""+str(source)+"\" target=\""+ str(target) + "\" weight=\""+ str(proximity) + "\"/>"

def save_to_file(filePath, gexfToSave):
	text_file = open(filePath, "w")
	text_file.write(gexfToSave)
	text_file.close()

def get_features(job_list, force_rescan):
	print "get_features"
	if(force_rescan == True):
		print ">force_rescan on " + str(len(job_list)) + " jobs"
		analysed = 0
		audiofile_features = {}
		for i in range(len(job_list)):
			try:
				print "> " + str(i) + " / " + str(len(job_list)),
				audiofile = AudioFile.open(job_list[i])
				print ",s",
				spectrum = audiofile.spectrum()				
				rms = audiofile.rms()
				print ",rms",
				centroid = spectrum.centroid
				print ",centroid",
				flatness = -1
				try:
					flatness = str(spectrum.flatness())
					pass
				except:
					pass
				finally:
					pass

				print ",flatness",				
				features = [rms, centroid, flatness]
				print ",done!"
				audiofile_features[job_list[i]] = features
				analysed = analysed + 1
			except:
				#print "Exception: " + job_list[i]
				pass
			else:
				pass
			finally:
				pass	

		print "<force_rescan " + str(analysed) + "/" + str(len(job_list)) + " scanned."

		rms = []
		centroid = []
		flatness = []

		for i in audiofile_features.keys():
			rms.append(audiofile_features[i][0])
			centroid.append(audiofile_features[i][1])
			flatness.append(float(audiofile_features[i][2]))

		#serialisation avec pickle 
		with open('/Users/neogaldr/cluster/rms.pkl', 'wb') as f:
			pkl.dump(rms,f)
		with open('/Users/neogaldr/cluster/centroid.pkl', 'wb') as f:
			pkl.dump(centroid,f)
		with open('/Users/neogaldr/cluster/flatness.pkl', 'wb') as f:
			pkl.dump(flatness,f)
		with open('/Users/neogaldr/cluster/features.pkl', 'wb') as f:
			pkl.dump(audiofile_features,f)			

	#ouvrir un serialized
	with open('/Users/neogaldr/Developments/Explosound/Cluster/rms.pkl', 'rb') as f:
		rms = pkl.load(f)
	with open('/Users/neogaldr/Developments/Explosound/Cluster/centroid.pkl', 'rb') as f:
		centroid = pkl.load(f)
	with open('/Users/neogaldr/Developments/Explosound/Cluster/flatness.pkl', 'rb') as f:
		flatness = pkl.load(f)
	with open('/Users/neogaldr/Developments/Explosound/Cluster/features.pkl', 'rb') as f:
		features = pkl.load(f)

	#centrer et reduire
	ppr.scale(rms)
	ppr.scale(centroid)
	ppr.scale(flatness)

	filtered_features = []

	for i in range(len(rms)):
		filtered_features.append([job_list[i], rms[i], centroid[i], flatness[i]])

	return filtered_features

def get_similarity(features):
	# TODO generate similarity matrix
	return 0




#MAIN ======================================================================================
folderPath = sys.argv[1]
defaultSize = 10
defaultR = 153
defaultG = 153
defaultB = 153
minX = -500
maxX = 500
minY = -500
maxY = 500
minZ = -10
maxZ = 10
edgeCreationProbability = 0.0013
edgeDefaultProximity = 1
forceRescan = False

print "Hello =)"
print "Generating job list for folder " + folderPath
job_list = make_job_list(folderPath)
print "Generating similarity matrix"
features = get_features(job_list, forceRescan)

print features

similarity = get_similarity(features)

print "____________________________"
print "Generating the resulting graph"
print ">nodes"
# Generate all nodes
node_list = []
id = 0
for job in make_job_list(folderPath):
	node_list.append(get_gexf_node(id, job, 10, random.uniform(minX, maxX), random.uniform(minY, maxY), random.uniform(minZ, maxZ),\
		defaultR, defaultG, defaultB))
	id += 1

print ">edges"
# Generate all edges
edge_list = []
id = 0
for nodeId1 in range(0, len(node_list)):
	for nodeId2 in range(0, len(node_list)):
		if(random.uniform(0, 1) < edgeCreationProbability):
			edge_list.append(get_gexf_edge(id, nodeId1, nodeId2, edgeDefaultProximity))
			id+=1

print ">gexf header and footer"
# Generate the gexf
gexf_file = get_gexf_header()

{'fraction_non_alphanumeric': 0.1774633481077395, 'fraction_numerical': 0.015683600409137403, 'mean_word_length': 4.018391787852866, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 2, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '69187', 'n_tokens_mistral': 4695, 'n_tokens_neox': 4190, 'n_words': 975}