<commit_before>/*
 * Copyright (c) 2015 Snowplow Analytics Ltd. All rights reserved.
 *
 * This program is licensed to you under the Apache License Version 2.0,
 * and you may not use this file except in compliance with the Apache License Version 2.0.
 * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the Apache License Version 2.0 is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 */
package com.snowplowanalytics.schemaguru
package cli

// Scalaz
import scalaz._
import Scalaz._

// Java
import java.io.File

// Argot
import org.clapper.argot._
import ArgotConverters._

// Schema DDL
import com.snowplowanalytics.schemaddl.{ FlatSchema, SelfDescInfo }
import com.snowplowanalytics.schemaddl.generators.{
  SchemaFlattener => SF
}
import com.snowplowanalytics.schemaddl.generators.redshift.{
  JsonPathGenerator => JPG,
  RedshiftDdlGenerator => RDG,
  Ddl
}
import com.snowplowanalytics.schemaddl.utils.{ StringUtils => SU }

// This library
import utils.{
  FileUtils => FU,
  FileSystemJsonGetters
}

/**
 * Holds all information passed with CLI and decides how to produce
 * DDL and JSON Paths
 *
 * @param args array of arguments passed via CLI
 */
class DdlCommand(val args: Array[String]) extends FileSystemJsonGetters {
  import DdlCommand._

  // Required
  val inputArgument = parser.parameter[File]("input", "Path to schema or directory with schemas", false)

  // Set all arguments
  val outputOption = parser.option[File](List("output"), "path", "Destination directory")
  val dbOption = parser.option[String](List("db"), "name", "For which DB we need to produce DDL (default: redshift)")
  val withJsonPathsFlag = parser.flag("with-json-paths", false, "Produce JSON Paths files with DDL")
  val rawModeFlag = parser.flag("raw", false, "Produce raw DDL without Snowplow-specific data")
  val schemaOption = parser.option[String](List("schema"), "name", "Redshift schema name")
  val sizeOption = parser.option[Int](List("varchar-size"), "n", "Default size for varchar data type")
  val splitProductFlag = parser.flag("split-product", false, "Split product types into different keys")

  parser.parse(args)

  // Get all arguments
  val input = inputArgument.value.get  // isn't optional
  val outputPath = outputOption.value.getOrElse(new File("."))
  val db = dbOption.value.getOrElse("redshift")
  val withJsonPaths = withJsonPathsFlag.value.getOrElse(false)
  val rawMode = rawModeFlag.value.getOrElse(false)
  val schemaName = if (!rawMode && schemaOption.value.isEmpty) { Some("atomic") } else { schemaOption.value }
  val size = sizeOption.value.getOrElse(255)
  val splitProduct = splitProductFlag.value.getOrElse(false)

  val schemaList: ValidJsonFileList =
    if (input.isDirectory) {
      getJsonFilesFromFolder(input)
    } else {
      List(getJsonFileFromFile(input))
    }

  (withJsonPaths, splitProduct) match {
    case (true, true) => parser.usage("Options --with-json-paths and --split-product can't be used together")
    case _            =>
  }

  schemaList match {
    case Nil       => parser.usage(s"Directory ${input.getAbsolutePath} does not contain any JSON files")
    case someJsons => someJsons.map(processAndOutput)
  }

  /**
   * Process schema and output JSON Path and DDL
   *
   * @param file file with JSON Schema
   */
  private def processAndOutput(file: Validation[String, JsonFile]): Unit = {
    processSchema(file) match {
      case Success(ddlOutput) => output(ddlOutput)
      case Failure(str) => {
        println(str)
        sys.exit(1)<commit_msg>Change default varchar size to 4096 (close snowplow/schema-ddl#30)<commit_after>/*
 * Copyright (c) 2015 Snowplow Analytics Ltd. All rights reserved.
 *
 * This program is licensed to you under the Apache License Version 2.0,
 * and you may not use this file except in compliance with the Apache License Version 2.0.
 * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the Apache License Version 2.0 is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 */
package com.snowplowanalytics.schemaguru
package cli

// Scalaz
import scalaz._
import Scalaz._

// Java
import java.io.File

// Argot
import org.clapper.argot._
import ArgotConverters._

// Schema DDL
import com.snowplowanalytics.schemaddl.{ FlatSchema, SelfDescInfo }
import com.snowplowanalytics.schemaddl.generators.{
  SchemaFlattener => SF
}
import com.snowplowanalytics.schemaddl.generators.redshift.{
  JsonPathGenerator => JPG,
  RedshiftDdlGenerator => RDG,
  Ddl
}
import com.snowplowanalytics.schemaddl.utils.{ StringUtils => SU }

// This library
import utils.{
  FileUtils => FU,
  FileSystemJsonGetters
}

/**
 * Holds all information passed with CLI and decides how to produce
 * DDL and JSON Paths
 *
 * @param args array of arguments passed via CLI
 */
class DdlCommand(val args: Array[String]) extends FileSystemJsonGetters {
  import DdlCommand._

  // Required
  val inputArgument = parser.parameter[File]("input", "Path to schema or directory with schemas", false)

  // Set all arguments
  val outputOption = parser.option[File](List("output"), "path", "Destination directory")
  val dbOption = parser.option[String](List("db"), "name", "For which DB we need to produce DDL (default: redshift)")
  val withJsonPathsFlag = parser.flag("with-json-paths", false, "Produce JSON Paths files with DDL")
  val rawModeFlag = parser.flag("raw", false, "Produce raw DDL without Snowplow-specific data")
  val schemaOption = parser.option[String](List("schema"), "name", "Redshift schema name")
  val sizeOption = parser.option[Int](List("varchar-size"), "n", "Default size for varchar data type")
  val splitProductFlag = parser.flag("split-product", false, "Split product types into different keys")

  parser.parse(args)

  // Get all arguments
  val input = inputArgument.value.get  // isn't optional
  val outputPath = outputOption.value.getOrElse(new File("."))
  val db = dbOption.value.getOrElse("redshift")
  val withJsonPaths = withJsonPathsFlag.value.getOrElse(false)
  val rawMode = rawModeFlag.value.getOrElse(false)
  val schemaName = if (!rawMode && schemaOption.value.isEmpty) { Some("atomic") } else { schemaOption.value }
  val size = sizeOption.value.getOrElse(4096)
  val splitProduct = splitProductFlag.value.getOrElse(false)

  val schemaList: ValidJsonFileList =
    if (input.isDirectory) {
      getJsonFilesFromFolder(input)
    } else {
      List(getJsonFileFromFile(input))
    }

  (withJsonPaths, splitProduct) match {
    case (true, true) => parser.usage("Options --with-json-paths and --split-product can't be used together")
    case _            =>
  }

  schemaList match {
    case Nil       => parser.usage(s"Directory ${input.getAbsolutePath} does not contain any JSON files")
    case someJsons => someJsons.map(processAndOutput)
  }

  /**
   * Process schema and output JSON Path and DDL
   *
   * @param file file with JSON Schema
   */
  private def processAndOutput(file: Validation[String, JsonFile]): Unit = {
    processSchema(file) match {
      case Success(ddlOutput) => output(ddlOutput)
      case Failure(str) => {
        println(str)
        sys.exit(1)
{'fraction_non_alphanumeric': 0.10412903225806451, 'fraction_numerical': 0.006064516129032258, 'mean_word_length': 4.052803129074316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5528194', 'n_tokens_mistral': 2360, 'n_tokens_neox': 2274, 'n_words': 734}