<commit_before>import datetime
from pyspark.sql.types import *

def load_wiki_rdd(sc, path):
    def parse_line(line):
        tokens = line.split('\t')
        timestamp = datetime.datetime(int(tokens[0]), int(tokens[1]), int(tokens[2]), int(tokens[3]))
        page = tokens[4]
        views = int(tokens[5])
        return (timestamp, page, views)
    return sc.textFile(path).map(parse_line)

def load_wiki_df(sql_context, path):
    rdd = load_wiki_rdd(sql_context._sc, path)
    fields = [ \
        StructField('timestamp', TimestampType(), True), \
        StructField('page', StringType(), True), \
        StructField('views', IntegerType(), True)]
    schema = StructType(fields)
    return sql_context.createDataFrame(rdd, schema)

def load_ticker_rdd(sc, path):
    def parse_line(line):
        tokens = line.split('\t')
        timestamp = datetime.datetime(int(tokens[0]), int(tokens[1]), int(tokens[2]), int(tokens[3]))
        symbol = tokens[4]
        price = float(tokens[6])
        return (timestamp, symbol, price)

    return sc.textFile(path).map(parse_line)

def load_ticker_df(sqlContext, path):
    rdd = load_ticker_rdd(sqlContext._sc, path)
    fields = [ \
        StructField('timestamp', TimestampType(), True), \
        StructField('page', StringType(), True), \
        StructField('views', DoubleType(), True)]
    schema = StructType(fields)
    return sql_context.createDataFrame(rdd, schema)
<commit_msg>Fixed column names in loaddata<commit_after>import datetime
from pyspark.sql.types import *

def load_wiki_rdd(sc, path):
    def parse_line(line):
        tokens = line.split('\t')
        timestamp = datetime.datetime(int(tokens[0]), int(tokens[1]), int(tokens[2]), int(tokens[3]))
        page = tokens[4]
        views = int(tokens[5])
        return (timestamp, page, views)
    return sc.textFile(path).map(parse_line)

def load_wiki_df(sql_context, path):
    rdd = load_wiki_rdd(sql_context._sc, path)
    fields = [ \
        StructField('timestamp', TimestampType(), True), \
        StructField('page', StringType(), True), \
        StructField('views', IntegerType(), True)]
    schema = StructType(fields)
    return sql_context.createDataFrame(rdd, schema)

def load_ticker_rdd(sc, path):
    def parse_line(line):
        tokens = line.split('\t')
        timestamp = datetime.datetime(int(tokens[0]), int(tokens[1]), int(tokens[2]), int(tokens[3]))
        symbol = tokens[4]
        price = float(tokens[6])
        return (timestamp, symbol, price)

    return sc.textFile(path).map(parse_line)

def load_ticker_df(sqlContext, path):
    rdd = load_ticker_rdd(sqlContext._sc, path)
    fields = [ \
        StructField('timestamp', TimestampType(), True), \
        StructField('symbol', StringType(), True), \
        StructField('price', DoubleType(), True)]
    schema = StructType(fields)
    return sql_context.createDataFrame(rdd, schema)

{'fraction_non_alphanumeric': 0.14575455482983843, 'fraction_numerical': 0.008250257820556892, 'mean_word_length': 3.5539906103286385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1516539', 'n_tokens_mistral': 911, 'n_tokens_neox': 911, 'n_words': 186}