<commit_before>from pyspark import SparkContext
from pyspark.sql import DataFrame, SQLContext
from pyspark.sql import Row
import tensorflow as tf

import tensorframes as tfs
from tensorframes.core import _java_api

class TestCore(object):

    @classmethod
    def setup_class(cls):
        print "setup ", cls
        cls.sc = SparkContext('local[1]', cls.__name__)

    @classmethod
    def teardown_class(cls):
        print "teardown ", cls
        cls.sc.stop()

    def setUp(self):
        self.sql = SQLContext(TestCore.sc)
        self.api = _java_api()
        self.api.initialize_logging()
        print "setup"


    def teardown(self):
        print "teardown"

    def test_schema(self):
        data = [Row(x=float(x)) for x in range(100)]
        df = self.sql.createDataFrame(data)
        tfs.print_schema(df)

    def test_map_blocks_1(self):
        data = [Row(x=float(x)) for x in range(10)]
        df = self.sql.createDataFrame(data)
        with tf.Graph().as_default() as g:
            # The placeholder that corresponds to column 'x'
            x = tf.placeholder(tf.double, shape=[None], name="x")
            # The output that adds 3 to x
            z = tf.add(x, 3, name='z')
            # The resulting dataframe
            df2 = tfs.map_blocks(z, df)
        data2 = df2.collect()
        assert data2[0].z == 3.0, data2

    def test_map_rows_1(self):
        data = [Row(x=float(x)) for x in range(5)]
        df = self.sql.createDataFrame(data)
        with tf.Graph().as_default() as g:
            # The placeholder that corresponds to column 'x'
            x = tf.placeholder(tf.double, shape=[], name="x")
            # The output that adds 3 to x
            z = tf.add(x, 3, name='z')
            # The resulting dataframe<commit_msg>cleanup<commit_after>
from __future__ import print_function

from pyspark import SparkContext
from pyspark.sql import DataFrame, SQLContext
from pyspark.sql import Row
import tensorflow as tf

import tensorframes as tfs
from tensorframes.core import _java_api

class TestCore(object):

    @classmethod
    def setup_class(cls):
        print("setup ", cls)
        cls.sc = SparkContext('local[1]', cls.__name__)

    @classmethod
    def teardown_class(cls):
        print "teardown ", cls
        cls.sc.stop()

    def setUp(self):
        self.sql = SQLContext(TestCore.sc)
        self.api = _java_api()
        self.api.initialize_logging()
        print("setup")


    def teardown(self):
        print("teardown")

    def test_schema(self):
        data = [Row(x=float(x)) for x in range(100)]
        df = self.sql.createDataFrame(data)
        tfs.print_schema(df)

    def test_map_blocks_1(self):
        data = [Row(x=float(x)) for x in range(10)]
        df = self.sql.createDataFrame(data)
        with tf.Graph().as_default() as g:
            # The placeholder that corresponds to column 'x'
            x = tf.placeholder(tf.double, shape=[None], name="x")
            # The output that adds 3 to x
            z = tf.add(x, 3, name='z')
            # The resulting dataframe
            df2 = tfs.map_blocks(z, df)
        data2 = df2.collect()
        assert data2[0].z == 3.0, data2

    def test_map_rows_1(self):
        data = [Row(x=float(x)) for x in range(5)]
        df = self.sql.createDataFrame(data)
        with tf.Graph().as_default() as g:
            # The placeholder that corresponds to column 'x'
            x = tf.placeholder(tf.double, shape=[], name="x")
            # The output that adds 3 to x
            z = tf.add(x, 3, name='z')
            # The resulting dataframe
{'fraction_non_alphanumeric': 0.11398387545176536, 'fraction_numerical': 0.011676396997497914, 'mean_word_length': 2.270909090909091, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4217218', 'n_tokens_mistral': 1159, 'n_tokens_neox': 1128, 'n_words': 309}