<commit_before>{
  "defaultSparkStreamingWindow": 6000,
  "defaultStorageLevel": "MEMORY_AND_DISK_SER",
  "defaultCheckpointPath": "/tmp/checkpoint",
  "defaultPartitionFormat": "day",
  "steps": [
    {
      "name": "_POLICY_._STEPS_._DESCRIPTION_",
      "icon": "icon-tag_left"
    },
    {
      "name": "_POLICY_._STEPS_._INPUT_",
      "icon": "icon-import"
    },
    {
      "name": "_POLICY_._STEPS_._MODEL_",
      "icon": "icon-content-left"
    },
    {
      "name": "_POLICY_._STEPS_._CUBES_",
      "icon": "icon-box"
    },
    {
      "name": "_POLICY_._STEPS_._OUTPUTS_",
      "icon": "icon-export"
    },
    {
      "name": "_POLICY_._STEPS_._FINISH_",
      "icon": "icon-paper"
    }
  ],
  "sparkStreamingWindow": {
    "min": 0,
    "max": 10000
  },
  "checkpointInterval": {
    "min": 0,
    "max": 10000
  },
  "checkpointAvailability": {
    "min": 0,
    "max": 10000
  },
  "partitionFormat": {
    "values": [
      {
        "label": "year",
        "value": "year"
      },
      {
        "label": "month",
        "value": "month"
      },
      {
        "label": "day",
        "value": "day"
      },
      {
        "label": "hour",
        "value": "hour"
      },
      {
        "label": "minute",
        "value": "minute"
      }
    ]
  },
  "storageLevel": {
    "values": [
      {
        "label": "DISK_ONLY",
        "value": "DISK_ONLY"
      },
      {
        "label": "DISK_ONLY_2",
        "value": "DISK_ONLY_2"
      },
      {
        "label": "MEMORY_ONLY",
        "value": "MEMORY_ONLY"
      },
      {
        "label": "MEMORY_ONLY_2",
        "value": "MEMORY_ONLY_2"
      },
      {
        "label": "MEMORY_ONLY_SER",
        "value": "MEMORY_ONLY_SER"
      },
      {
        "label": "MEMORY_ONLY_SER_2",
        "value": "MEMORY_ONLY_SER_2"
      },
      {
        "label": "MEMORY_AND_DISK",
        "value": "MEMORY_AND_DISK"
      },
      {
        "label": "MEMORY_AND_DISK_2",
        "value": "MEMORY_AND_DISK_2"
      },
      {
        "label": "MEMORY_AND_DISK_SER",
        "value": "MEMORY_AND_DISK_SER"
      },
      {
        "label": "MEMORY_AND_DISK_SER_2",
        "value": "MEMORY_AND_DISK_SER_2"
      }
    ]
  },
  "helpLinks": {
    "description": "http://docs.stratio.com/modules/sparkta/development/policy.html#general-configuration",
    "inputs": "http://docs.stratio.com/modules/sparkta/development/policy.html#inputs",
    "models": "http://docs.stratio.com/modules/sparkta/development/policy.html#transformations",
    "cubes": "http://docs.stratio.com/modules/sparkta/development/policy.html#cubes",
    "outputs": "http://docs.stratio.com/modules/sparkta/development/policy.html#outputs"
  },
  "configPlaceholder": "{    \n    \"inputFormat\": \"unixMillis\", \n   \"dbName\": \"sparkta\"\n}",
  "configurationHelpLink": "http://docs.stratio.com/modules/sparkta/development/transformations.html",
  "morphlinesDefaultConfiguration": {
    "morphline": {
      "id": "morphline1",
      "importCommands": [
        "org.kitesdk.**"
      ],
      "commands": [
        {
          "readJson": {}<commit_msg>fix(issue sparkta-113): broken link in policy description<commit_after>{
  "defaultSparkStreamingWindow": 6000,
  "defaultStorageLevel": "MEMORY_AND_DISK_SER",
  "defaultCheckpointPath": "/tmp/checkpoint",
  "defaultPartitionFormat": "day",
  "steps": [
    {
      "name": "_POLICY_._STEPS_._DESCRIPTION_",
      "icon": "icon-tag_left"
    },
    {
      "name": "_POLICY_._STEPS_._INPUT_",
      "icon": "icon-import"
    },
    {
      "name": "_POLICY_._STEPS_._MODEL_",
      "icon": "icon-content-left"
    },
    {
      "name": "_POLICY_._STEPS_._CUBES_",
      "icon": "icon-box"
    },
    {
      "name": "_POLICY_._STEPS_._OUTPUTS_",
      "icon": "icon-export"
    },
    {
      "name": "_POLICY_._STEPS_._FINISH_",
      "icon": "icon-paper"
    }
  ],
  "sparkStreamingWindow": {
    "min": 0,
    "max": 10000
  },
  "checkpointInterval": {
    "min": 0,
    "max": 10000
  },
  "checkpointAvailability": {
    "min": 0,
    "max": 10000
  },
  "partitionFormat": {
    "values": [
      {
        "label": "year",
        "value": "year"
      },
      {
        "label": "month",
        "value": "month"
      },
      {
        "label": "day",
        "value": "day"
      },
      {
        "label": "hour",
        "value": "hour"
      },
      {
        "label": "minute",
        "value": "minute"
      }
    ]
  },
  "storageLevel": {
    "values": [
      {
        "label": "DISK_ONLY",
        "value": "DISK_ONLY"
      },
      {
        "label": "DISK_ONLY_2",
        "value": "DISK_ONLY_2"
      },
      {
        "label": "MEMORY_ONLY",
        "value": "MEMORY_ONLY"
      },
      {
        "label": "MEMORY_ONLY_2",
        "value": "MEMORY_ONLY_2"
      },
      {
        "label": "MEMORY_ONLY_SER",
        "value": "MEMORY_ONLY_SER"
      },
      {
        "label": "MEMORY_ONLY_SER_2",
        "value": "MEMORY_ONLY_SER_2"
      },
      {
        "label": "MEMORY_AND_DISK",
        "value": "MEMORY_AND_DISK"
      },
      {
        "label": "MEMORY_AND_DISK_2",
        "value": "MEMORY_AND_DISK_2"
      },
      {
        "label": "MEMORY_AND_DISK_SER",
        "value": "MEMORY_AND_DISK_SER"
      },
      {
        "label": "MEMORY_AND_DISK_SER_2",
        "value": "MEMORY_AND_DISK_SER_2"
      }
    ]
  },
  "helpLinks": {
    "description": "http://docs.stratio.com/modules/sparkta/0.6/usingSparkta.html#policy",
    "inputs": "http://docs.stratio.com/modules/sparkta/development/policy.html#inputs",
    "models": "http://docs.stratio.com/modules/sparkta/development/policy.html#transformations",
    "cubes": "http://docs.stratio.com/modules/sparkta/development/policy.html#cubes",
    "outputs": "http://docs.stratio.com/modules/sparkta/development/policy.html#outputs"
  },
  "configPlaceholder": "{    \n    \"inputFormat\": \"unixMillis\", \n   \"dbName\": \"sparkta\"\n}",
  "configurationHelpLink": "http://docs.stratio.com/modules/sparkta/development/transformations.html",
  "morphlinesDefaultConfiguration": {
    "morphline": {
      "id": "morphline1",
      "importCommands": [
        "org.kitesdk.**"
      ],
      "commands": [
        {
          "readJson": {}
{'fraction_non_alphanumeric': 0.18545163868904876, 'fraction_numerical': 0.01135091926458833, 'mean_word_length': 2.1124378109452735, 'pattern_counts': {'":': 152, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1990115', 'n_tokens_mistral': 2367, 'n_tokens_neox': 2299, 'n_words': 288}