<commit_before># ref <master> <dataFile> <outputDir> <mode>
# 
# compute summary statistics
#

import sys
import os
from numpy import *
import thunder
from thunder.util.dataio import *
from pyspark import SparkContext

argsIn = sys.argv[1:]

if len(argsIn) < 4:
  print >> sys.stderr, \
  "(ref) usage: ref <master> <dataFile> <outputDir> <mode>"
  exit(-1)

# parse inputs
sc = SparkContext(argsIn[0], "ref")
dataFile = str(argsIn[1])
outputDir = str(argsIn[2])
mode = str(argsIn[3])
if not os.path.exists(outputDir) : os.makedirs(outputDir)

# parse data
lines = sc.textFile(dataFile)
#data = parse(lines)
data = lines.map(lambda x : parseVector(x))

print(data.first())

# get z ordering
zinds = data.filter(lambda (k,x) : (k[0] == 1) & (k[1] == 1)).map(lambda (k,x) : k[2])
print(zinds.collect())
#saveout(zinds,outputDir,"zinds","matlab")

# compute ref
if mode == 'med':
	ref = data.map(lambda (k,x) : median(x))
if mode == 'mean':
	ref = data.map(lambda (k,x) : mean(x))<commit_msg>Testing<commit_after># ref <master> <dataFile> <outputDir> <mode>
# 
# compute summary statistics
#

import sys
import os
from numpy import *
import thunder
from thunder.util.dataio import *
from pyspark import SparkContext

argsIn = sys.argv[1:]

if len(argsIn) < 4:
  print >> sys.stderr, \
  "(ref) usage: ref <master> <dataFile> <outputDir> <mode>"
  exit(-1)

# parse inputs
sc = SparkContext(argsIn[0], "ref")
dataFile = str(argsIn[1])
outputDir = str(argsIn[2])
mode = str(argsIn[3])
if not os.path.exists(outputDir) : os.makedirs(outputDir)

# parse data
lines = sc.textFile(dataFile)
#data = parse(lines)
data = lines.map(lambda x : parseVector(x,"raw","xyz"))

print(data.first())

# get z ordering
zinds = data.filter(lambda (k,x) : (k[0] == 1) & (k[1] == 1)).map(lambda (k,x) : k[2])
print(zinds.collect())
#saveout(zinds,outputDir,"zinds","matlab")

# compute ref
if mode == 'med':
	ref = data.map(lambda (k,x) : median(x))
if mode == 'mean':
	ref = data.map(lambda (k,x) : mean(x))
{'fraction_non_alphanumeric': 0.18346774193548387, 'fraction_numerical': 0.012096774193548387, 'mean_word_length': 5.0703363914373085, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2203804', 'n_tokens_mistral': 784, 'n_tokens_neox': 770, 'n_words': 191}