<commit_before>from pyspark.sql import HiveContext
from pyspark.mllib.regression import LabeledPoint, LinearModel, LinearRegressionWithSGD, LassoWithSGD
from pyspark.ml.regression import LinearRegression
from pyspark.sql.functions import col, sum
from copy import deepcopy
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.evaluation import RegressionMetrics

sc = SparkContext()
sqlContext = HiveContext(sc)
# The races from the census data were normalized in order
qry = "SELECT AVG(success_metric) as success_metric, hispanic/population as hispanic_percentage, zipcode from census_rest_success group by 1, 2"

df = sqlContext.sql(qry)
#CITATION:
#http://stackoverflow.com/questions/33900726/count-number-of-non-nan-entries-in-each-column-of-spark-dataframe-with-pyspark
def count_not_null(c):
    return sum(col(c).isNotNull().cast("integer")).alias(c)

exprs = [count_not_null(c) for c in df.columns]
df.agg(*exprs).show()

df = df.dropna()


features = df.select(df['success_metric'], df['hispanic_percentage'])`

training, test = features.randomSplit([0.7, 0.3], seed=11L)

feats_train = training.collect()
train_dict = [i.asDict() for i in feats_train]

feats_test = test.collect()
test_dict = [i.asDict() for i in feats_test]

def parsePoint(d): ## wont be able to use line.split here?
    d_copy = deepcopy(d) # I hate using deepcopy so much
    pred = d_copy['success_metric']
    d.pop('success_metric', None)
    values = [float(x) for x in d.values()] ##this block is unusable until we have our Hive Data
    return (pred, Vectors.dense(values))

# training set
trainParsed = sc.parallelize(map(parsePoint, train_dict))
# test set
testParsed = sc.parallelize(map(parsePoint, test_dict))

trainDf = sqlContext.createDataFrame(trainParsed, ["label", "features"])<commit_msg>regression model for mexican restaurants<commit_after>from pyspark.sql import HiveContext
from pyspark.mllib.regression import LabeledPoint, LinearModel, LinearRegressionWithSGD, LassoWithSGD
from pyspark.ml.regression import LinearRegression
from pyspark.sql.functions import col, sum
from copy import deepcopy
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.evaluation import RegressionMetrics

sc = SparkContext()
sqlContext = HiveContext(sc)
# The races from the census data were normalized in order
qry = """SELECT AVG(success_metric) as success_metric,
         hispanic/population as hispanic_percentage, zipcode
         from census_rest_success where array_contains(categories, 'Mexican')
         group by hispanic, population, zipcode"""

df = sqlContext.sql(qry)
#CITATION:
#http://stackoverflow.com/questions/33900726/count-number-of-non-nan-entries-in-each-column-of-spark-dataframe-with-pyspark
def count_not_null(c):
    return sum(col(c).isNotNull().cast("integer")).alias(c)

exprs = [count_not_null(c) for c in df.columns]
df.agg(*exprs).show()

df = df.dropna()


features = df.select(df['success_metric'], df['hispanic_percentage'])

training, test = features.randomSplit([0.7, 0.3], seed=11L)

feats_train = training.collect()
train_dict = [i.asDict() for i in feats_train]

feats_test = test.collect()
test_dict = [i.asDict() for i in feats_test]

def parsePoint(d): ## wont be able to use line.split here?
    d_copy = deepcopy(d) # I hate using deepcopy so much
    pred = d_copy['success_metric']
    d.pop('success_metric', None)
    values = [float(x) for x in d.values()] ##this block is unusable until we have our Hive Data
    return (pred, Vectors.dense(values))

# training set
trainParsed = sc.parallelize(map(parsePoint, train_dict))
# test set
testParsed = sc.parallelize(map(parsePoint, test_dict))

trainDf = sqlContext.createDataFrame(trainParsed, ["label", "features"])
{'fraction_non_alphanumeric': 0.11437029063509149, 'fraction_numerical': 0.008073196986006458, 'mean_word_length': 5.498251748251748, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7567425', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1235, 'n_words': 319}