<commit_before>import sys
import os
from numpy import *
from numpy.linalg import *
from scipy.io import * 
from pyspark import SparkContext
import logging

def parseVector(line):
	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:]) # get tseries, drop x,y,z coords
	med = median(ts)
	ts = (ts - med) / (med) # convert to dff
	return ts

def closestPoint(p, centers, dist):
	bestIndex = 0
	closest = float("+inf")
	for i in range(len(centers)):
		if dist == 'euclidean':
			tempDist = sum((p - centers[i]) ** 2)
		if dist == 'corr':
			tempDist = max(1 - dot(p,centers[i]),0)
		if tempDist < closest:
			closest = tempDist
			bestIndex = i
	return (bestIndex, closest)

if len(sys.argv) < 6:
  print >> sys.stderr, \
  "(kmeans) usage: kmeans <master> <inputFile> <outputFile> <k> <dist>"
  exit(-1)

sc = SparkContext(sys.argv[1], "kmeans")
inputFile_X = str(sys.argv[2])
k = int(sys.argv[4])
dist = str(sys.argv[5])
outputFile = str(sys.argv[3]) + "-kmeans-" + str(k) + "-" + dist
if not os.path.exists(outputFile):
    os.makedirs(outputFile)
logging.basicConfig(filename=outputFile+'/'+'stdout.log',level=logging.INFO,format='%(asctime)s %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p')

logging.info("(kmeans) loading data")
lines_X = sc.textFile(inputFile_X)
X = lines_X.map(parseVector)
if dist == 'corr':
	X = X.map(lambda x : (x - mean(x)) / norm(x)).cache()

kPoints = X.take(k)
if dist == 'corr':
	kPoints = map(lambda x : x - mean(x), kPoints);

convergeDist = 0.001
tempDist = 1.0
iteration = 0
mxIteration = 100

while (tempDist > convergeDist) & (iteration < mxIteration):
	logging.info("(kmeans) starting iteration " + str(iteration))
	if dist == 'corr':
		kPointsNorm = map(lambda x : x / norm(x), kPoints)
		closest = X.map(<commit_msg>Bug fixes<commit_after>import sys
import os
from numpy import *
from numpy.linalg import *
from scipy.io import * 
from pyspark import SparkContext
import logging

def parseVector(line):
	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:]) # get tseries, drop x,y,z coords
	med = median(ts)
	ts = (ts - med) / (med) # convert to dff
	return ts

def closestPoint(p, centers, dist):
	bestIndex = 0
	closest = float("+inf")
	for i in range(len(centers)):
		if dist == 'euclidean':
			tempDist = sum((p - centers[i]) ** 2)
		if dist == 'corr':
			tempDist = max(1 - dot(p,centers[i]),0)
		if tempDist < closest:
			closest = tempDist
			bestIndex = i
	return (bestIndex, closest)

if len(sys.argv) < 6:
  print >> sys.stderr, \
  "(kmeans) usage: kmeans <master> <inputFile> <outputFile> <k> <dist>"
  exit(-1)

sc = SparkContext(sys.argv[1], "kmeans")
inputFile_X = str(sys.argv[2])
k = int(sys.argv[4])
dist = str(sys.argv[5])
outputFile = str(sys.argv[3]) + "-kmeans-" + str(k) + "-" + dist
if not os.path.exists(outputFile):
    os.makedirs(outputFile)
logging.basicConfig(filename=outputFile+'/'+'stdout.log',level=logging.INFO,format='%(asctime)s %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p')

logging.info("(kmeans) loading data")
lines_X = sc.textFile(inputFile_X)

if dist == 'corr':
	X = lines_X.map(parseVector)
	X = X.map(lambda x : (x - mean(x)) / norm(x)).cache()
elif dist == 'euclidean'
	X = lines_X.map(parseVector).cache()

kPoints = X.take(k)
if dist == 'corr':
	kPoints = map(lambda x : x - mean(x), kPoints);

convergeDist = 0.001
tempDist = 1.0
iteration = 0
mxIteration = 100

while (tempDist > convergeDist) & (iteration < mxIteration):
	logging.info("(kmeans) starting iteration " + str(iteration))
	if dist == 'corr':
		kPointsNorm = map(lambda x : x / norm(x), kPoints)
		closest = X.map(
{'fraction_non_alphanumeric': 0.1773247696174253, 'fraction_numerical': 0.012287070650656241, 'mean_word_length': 4.330357142857143, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5169312', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1379, 'n_words': 347}