<commit_before># ref <master> <inputFile> <inds> <outputFile>
# 
# compute summary statistics on an xyz stack
# each row is (x,y,z,timeseries)
#

import sys
import os
from numpy import *
from scipy.linalg import *
from scipy.sparse import *
from scipy.io import * 
from pyspark import SparkContext
import logging

if len(sys.argv) < 4:
  print >> sys.stderr, \
  "(ref) usage: ref <master> <inputFileX> <inputFileY> <outputFile> <mode>"
  exit(-1)

def parseVector(line):
	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:]) # get tseries
	return ((int(vec[0]),int(vec[1]),int(vec[2])),ts) # (x,y,z),(tseries) pair 

# parse inputs
sc = SparkContext(sys.argv[1], "ref")
inputFile_X = str(sys.argv[2])
inputFile_y = str(sys.argv[3])
outputFile = str(sys.argv[4])
mode = str(sys.argv[5])
logging.basicConfig(filename=outputFile+'stdout.log',level=logging.INFO,format='%(asctime)s %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p')

# parse data
logging.info("(ref) loading data")
y = loadmat(inputFile_y)['y']
y = y.astype(float)
inds = sum(y,axis=0)!=0
y = y[:,inds] 
lines_X = sc.textFile(inputFile_X) # the data
X = lines_X.map(lambda x : parseVector(x,inds)).cache()

# get z ordering
logging.info("(ref) getting z ordering")
zinds = X.filter(lambda (k,x) : (k[0] == 1) & (k[1] == 1)).map(lambda (k,x) : k[2]).collect()
savemat(outputFile+"zinds.mat",mdict={'zinds':zinds},oned_as='column',do_compression='true')

# compute ref
logging.info('(ref) computing reference image')
if mode == 'med':
	ref = X.map(lambda (k,x) : median(x)).collect()
if mode == 'mean':
	ref = X.map(lambda (k,x) : mean(x)).collect()
if mode == 'std':
	ref = X.map(lambda (k,x) : std((x - mean(x))/(mean(x)+0.1))).collect()
if mode == 'perc':
	ref = X.map(lambda (k,x) : percentile(x,90)).collect()
logging.info('(ref) saving results...')
savemat(outputFile+mode+".mat",mdict={'ref':ref},oned_as='column',do_compression='true')<commit_msg>Bug fixes<commit_after># ref <master> <inputFile> <inds> <outputFile>
# 
# compute summary statistics on an xyz stack
# each row is (x,y,z,timeseries)
#

import sys
import os
from numpy import *
from scipy.linalg import *
from scipy.sparse import *
from scipy.io import * 
from pyspark import SparkContext
import logging

if len(sys.argv) < 4:
  print >> sys.stderr, \
  "(ref) usage: ref <master> <inputFileX> <inputFileY> <outputFile> <mode>"
  exit(-1)

def parseVector(line,inds):
	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:]) # get tseries
	ts = ts[inds]
	return ((int(vec[0]),int(vec[1]),int(vec[2])),ts) # (x,y,z),(tseries) pair 

# parse inputs
sc = SparkContext(sys.argv[1], "ref")
inputFile_X = str(sys.argv[2])
inputFile_y = str(sys.argv[3])
outputFile = str(sys.argv[4])
mode = str(sys.argv[5])
logging.basicConfig(filename=outputFile+'stdout.log',level=logging.INFO,format='%(asctime)s %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p')

# parse data
logging.info("(ref) loading data")
y = loadmat(inputFile_y)['y']
y = y.astype(float)
inds = sum(y,axis=0)!=0
lines_X = sc.textFile(inputFile_X) # the data
X = lines_X.map(lambda x : parseVector(x,inds)).cache()

# get z ordering
logging.info("(ref) getting z ordering")
zinds = X.filter(lambda (k,x) : (k[0] == 1) & (k[1] == 1)).map(lambda (k,x) : k[2]).collect()
savemat(outputFile+"zinds.mat",mdict={'zinds':zinds},oned_as='column',do_compression='true')

# compute ref
logging.info('(ref) computing reference image')
if mode == 'med':
	ref = X.map(lambda (k,x) : median(x)).collect()
if mode == 'mean':
	ref = X.map(lambda (k,x) : mean(x)).collect()
if mode == 'std':
	ref = X.map(lambda (k,x) : std((x - mean(x))/(mean(x)+0.1))).collect()
if mode == 'perc':
	ref = X.map(lambda (k,x) : percentile(x,90)).collect()
logging.info('(ref) saving results...')
savemat(outputFile+mode+".mat",mdict={'ref':ref},oned_as='column',do_compression='true')
{'fraction_non_alphanumeric': 0.21155347384855583, 'fraction_numerical': 0.011449388498568826, 'mean_word_length': 5.767605633802817, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4573919', 'n_tokens_mistral': 1553, 'n_tokens_neox': 1531, 'n_words': 326}