<commit_before>package com.memsql.spark.interface.supportClasses

import com.memsql.spark.etl.api._
import com.memsql.spark.etl.utils.{SimpleJsonSchema, PhaseLogger}
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.types.{StructType, StructField, IntegerType}
import org.apache.spark.sql.{DataFrame, SQLContext}

/**
 * Created by marin on 10/1/15.
 */
class DummyExtractor extends SimpleByteArrayExtractor {
  override def nextRDD(sparkContext: SparkContext, config: UserExtractConfig, batchInterval: Long, logger: PhaseLogger): Option[RDD[Array[Byte]]] = None
}


class DummyTransformer extends SimpleByteArrayTransformer {
  override def transform(sqlContext: SQLContext, rdd: RDD[Array[Byte]], transformConfig: UserTransformConfig,
                         logger: PhaseLogger): DataFrame = null
}<commit_msg>Deleted comment.<commit_after>package com.memsql.spark.interface.supportClasses

import com.memsql.spark.etl.api._
import com.memsql.spark.etl.utils.{SimpleJsonSchema, PhaseLogger}
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.types.{StructType, StructField, IntegerType}
import org.apache.spark.sql.{DataFrame, SQLContext}

class DummyExtractor extends SimpleByteArrayExtractor {
  override def nextRDD(sparkContext: SparkContext, config: UserExtractConfig, batchInterval: Long, logger: PhaseLogger): Option[RDD[Array[Byte]]] = None
}


class DummyTransformer extends SimpleByteArrayTransformer {
  override def transform(sqlContext: SQLContext, rdd: RDD[Array[Byte]], transformConfig: UserTransformConfig,
                         logger: PhaseLogger): DataFrame = null
}
{'fraction_non_alphanumeric': 0.09970326409495549, 'fraction_numerical': 0.002967359050445104, 'mean_word_length': 5.637795275590551, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5221480', 'n_tokens_mistral': 512, 'n_tokens_neox': 494, 'n_words': 106}