<commit_before>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package com.drakeconsulting.big_data_maker

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
import org.apache.spark.sql.SQLContext
import java.io.File

object SimpleApp {
  def main(args: Array[String]) {
    case class Config(outDir: File = new File("."))
    val parser = new scopt.OptionParser[Config]("Big Data Maker") {
      head("Big Data Maker", "1.0")
      opt[File]('o', "outDir").required.action( (x,c) => c.copy(outDir = x) ).text("outDir is a directory")
    }

    var outDir = ""
    parser.parse(args, Config()) match {
      case Some(config) =>
        outDir = config.outDir.toString

      case None =>
        println("Oops.")
        System.exit(1)
    }

    println("outDir=" + outDir)
    val conf = new SparkConf().setAppName("Big Data Maker Application")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)

    val bigData = new BigData(sqlContext, outDir, 10, 1000)
    bigData.addColumn(new StringConstant("f1", "testing"))
    bigData.addColumn(new RandomLong("f2", 100000000000L))
    bigData.addColumn(new RandomLong("f3", 10000000000L))
    bigData.addColumn(new RandomLong("f4", 1000000000L))
    bigData.addColumn(new RandomDouble("f5", 1000000000.0))
    bigData.addColumn(new RandomDouble("f6", 100000000.0))
    bigData.addColumn(new RandomDouble("f7", 10000000.0))

    bigData.writeFile

  }
}
<commit_msg>pass numPartitions/rows on command-line<commit_after>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package com.drakeconsulting.big_data_maker

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
import org.apache.spark.sql.SQLContext
import java.io.File

object SimpleApp {
  def main(args: Array[String]) {

    case class Config(outDir: File = new File("."), numPartitions: Integer = 0, numRows: Integer = 0)

    val parser = new scopt.OptionParser[Config]("Big Data Maker") {
      head("Big Data Maker", "1.0")
      opt[File]('o', "outDir").required.action( (x,c) => c.copy(outDir = x) ).text("outDir is a directory")
      opt[Int]('p', "numPartitions").action( (x,c) => c.copy(numPartitions = x) ).text("# of partitions")
      opt[Int]('r', "numRows").action( (x,c) => c.copy(numRows = x) ).text("# of rows/partition")
    }

    var outDir = ""
    var numPartitions: Integer = 0
    var numRows: Integer = 0

    parser.parse(args, Config()) match {
      case Some(config) =>
        outDir = config.outDir.toString
        numPartitions = config.numPartitions
        numRows = config.numRows

      case None =>
        println("Oops.")
        System.exit(1)
    }

    if (numPartitions == 0) {
      numPartitions = 10
    }
    if (numRows == 0) {
      numRows = 1000
    }

    println("outDir=" + outDir)
    println("numPartitions=" + numPartitions)
    println("numRows=" + numRows)

    val conf = new SparkConf().setAppName("Big Data Maker Application")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)

    val bigData = new BigData(sqlContext, outDir, numPartitions, numRows)
    bigData.addColumn(new StringConstant("f1", "testing"))
    bigData.addColumn(new RandomLong("f2", 100000000000L))
    bigData.addColumn(new RandomLong("f3", 10000000000L))
    bigData.addColumn(new RandomLong("f4", 1000000000L))
    bigData.addColumn(new RandomDouble("f5", 1000000000.0))
    bigData.addColumn(new RandomDouble("f6", 100000000.0))
    bigData.addColumn(new RandomDouble("f7", 10000000.0))

    bigData.writeFile

  }
}

{'fraction_non_alphanumeric': 0.11097922848664689, 'fraction_numerical': 0.03402571711177053, 'mean_word_length': 3.8615384615384616, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 12, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3752360', 'n_tokens_mistral': 1671, 'n_tokens_neox': 1494, 'n_words': 479}