<commit_before>#!/usr/bin/env bash

# Copyright © 2016 <NAME>, Inc.
# 
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
# 
# http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
  
# Build script for docs

source ../vars
source ../_common/common-build.sh

CHECK_INCLUDES=${TRUE}

EXTRACT_TABLE_TOOL="../tools/docs-extract-table.py"

DOUBLE_RETURN_STRING="\

"
SINGLE_RETURN_STRING="\
"
TRIPLE_RETURN_STRING="\


"
RULE="${SINGLE_RETURN_STRING}---${DOUBLE_RETURN_STRING}"

# PRE_POST_RUN="pre-post-run"
PRE_POST_RUN="post-run-plugin"
NON_TRANSFORM_TYPES="action source sink ${PRE_POST_RUN}"
PLUGIN_TYPE_STRING="Hydrator Plugin Type:"
VERSION_STRING="Hydrator Version:"

function get_hydrator_version() {
  local base_target="${1}"
  local base_source="${2}"
  local source_url="${base_source}/pom.xml"
  local target="${base_target}/pom.xml"
  curl --silent ${source_url} --output ${target}
  HYDRATOR_VERSION=$(grep "<version>" ${target})
  HYDRATOR_VERSION=${HYDRATOR_VERSION#*<version>}
  HYDRATOR_VERSION=${HYDRATOR_VERSION%%</version>*}
  export HYDRATOR_VERSION
}

function download_md_file() {
  local source_dir="${1}"
  local source_file_name="${2}"
  local append_file="${3}"
  local plugin_type="${4}" # NOTE: singular types: "sink", not "sinks"
  local target_file_name="${5}"
  
  local source_url="${HYDRATOR_SOURCE}/${source_dir}/docs/${source_file_name}"

  local source_name="${source_file_name%-*}"
  local type=$(echo "${source_file_name#*-}" | tr [:upper:] [:lower:]) # batchsink
  type="${type%.md}" # strip suffix
  
  local type_capital="$(echo ${type:0:1} | tr [:lower:] [:upper:])${type:1}"
  if [[ "x${target_file_name}" == "x" ]]; then
    local target_file_name=$(echo "${source_file_name}" | tr [:upper:] [:lower:]) # cassandra-batchsink.md
  fi
  
  # Determine from name of the plugin file the:
  # type (source, sink, transform, shared-plugin, postaction)
  # Defining this in the parameters overrides this
  if [[ "x${plugin_type}" == "x" ]]; then
    if [[ "x${type:0:5}" == "xbatch" ]]; then
      plugin_type="${type:5}"
    elif [[ "x${type:0:8}" == "xrealtime" ]]; then
      plugin_type="${type:8}"
      
    # FIXME: this type "postaction" is going away
    elif [[ "x${type}" == "xpostaction" ]]; then
      plugin_type="post-run-plugin"
    # END FIXME
    
    # FIXME: these types "prerun" and  "postrun" are currently not used
#     elif [[ "x${type}" == "xprerun" ]]; then
#       plugin_type="pre-run"
#     elif [[ "x${type}" == "xpostrun" ]]; then
#       plugin_type="post-run"
    # END FIXME
    elif [[ "x${type}" == "xaction" ]]; then
      plugin_type="action"
    else
      # assume of type transform; to be copied to both batch and realtime
      plugin_type="transform"
    fi
  fi
  
  local target_dir="${plugin_type}s"

  echo ${NON_TRANSFORM_TYPES} | grep -q ${plugin_type}
  if [ "$?" == "0" ]; then
    local target_dir="${plugin_type}s"
  else
    local target_dir="transforms"
  fi

  local target="${BASE_TARGET}/${target_dir}/${target_file_name}"

  # Create display names for log output
  local fifty_spaces="                                                  "
  local display_source_file_name="${source_file_name}${fifty_spaces}"
  display_source_file_name="${display_source_file_name:0:50}"
  local display_source_dir="${source_dir}${fifty_spaces}"
  display_source_dir="${display_source_dir:0:25}"

  if curl --output /dev/null --silent --head --fail "${source_url}"; then
    echo "Downloading ${display_source_file_name} from ${display_source_dir} to ${target_dir}/${target_file_name}"
    #     Downloading Cassandra-batchsink.md from cassandra-plugins to batch/sink/cassandra.md
    curl --silent ${source_url} --output ${target}
    if [ -e "${target}" ]; then
      # If file does not begin with a "#" character, append "# title\n" to start
      local first=$(head -1 ${target})
      if [[ "x${first:0:2}" != "x# " ]]; then
        local m="Markdown file missing initial title: ${source_file_name}: ${source_name} ${type_capital}"
        echo_red_bold "${m}"
        set_message "${m}"
        echo "# ${source_name} ${type_capital}${DOUBLE_RETURN_STRING}$(cat ${target})" > ${target}
      fi
      if [[ "x${append_file}" != "x" ]]; then
        echo "  Appending ${append_file} to ${target_file_name}"
        cat ${BASE_TARGET}/${append_file} >> ${target}
      fi
      echo "${DOUBLE_RETURN_STRING}${RULE}- ${PLUGIN_TYPE_STRING} ${type}${DOUBLE_RETURN_STRING}- ${VERSION_STRING} ${HYDRATOR_VERSION}" >> ${target}
    else
      local m="File does not exist for ${target}"
      echo_red_bold "From ${source_url}"
      echo_red_bold "${m}"
      set_message "${m}"
    fi
  else
    local m="URL does not exist: ${source_url}"
    echo_red_bold "${m}"
    set_message "${m}"
  fi
}

function extract_table() {
  # Extracts a table written in Markdown from a file so it can be reused in reST files<commit_msg>Strip names from ends of titles.<commit_after>#!/usr/bin/env bash

# Copyright © 2016 <NAME>, Inc.
# 
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
# 
# http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
  
# Build script for docs

source ../vars
source ../_common/common-build.sh

CHECK_INCLUDES=${TRUE}

EXTRACT_TABLE_TOOL="../tools/docs-extract-table.py"

DOUBLE_RETURN_STRING="\

"
SINGLE_RETURN_STRING="\
"
TRIPLE_RETURN_STRING="\


"
RULE="${SINGLE_RETURN_STRING}---${DOUBLE_RETURN_STRING}"

# PRE_POST_RUN="pre-post-run"
PRE_POST_RUN="post-run-plugin"
NON_TRANSFORM_TYPES="action source sink ${PRE_POST_RUN}"
PLUGIN_TYPE_STRING="Hydrator Plugin Type:"
VERSION_STRING="Hydrator Version:"

function get_hydrator_version() {
  local base_target="${1}"
  local base_source="${2}"
  local source_url="${base_source}/pom.xml"
  local target="${base_target}/pom.xml"
  curl --silent ${source_url} --output ${target}
  HYDRATOR_VERSION=$(grep "<version>" ${target})
  HYDRATOR_VERSION=${HYDRATOR_VERSION#*<version>}
  HYDRATOR_VERSION=${HYDRATOR_VERSION%%</version>*}
  export HYDRATOR_VERSION
}

function download_md_file() {
  local source_dir="${1}"
  local source_file_name="${2}"
  local append_file="${3}"
  local plugin_type="${4}" # NOTE: singular types: "sink", not "sinks"
  local target_file_name="${5}"
  
  local source_url="${HYDRATOR_SOURCE}/${source_dir}/docs/${source_file_name}"

  local source_name="${source_file_name%-*}"
  local type=$(echo "${source_file_name#*-}" | tr [:upper:] [:lower:]) # batchsink
  type="${type%.md}" # strip suffix
  
  local type_capital="$(echo ${type:0:1} | tr [:lower:] [:upper:])${type:1}"
  if [[ "x${target_file_name}" == "x" ]]; then
    local target_file_name=$(echo "${source_file_name}" | tr [:upper:] [:lower:]) # cassandra-batchsink.md
  fi
  
  # Determine from name of the plugin file the:
  # type (source, sink, transform, shared-plugin, postaction)
  # Defining this in the parameters overrides this
  if [[ "x${plugin_type}" == "x" ]]; then
    if [[ "x${type:0:5}" == "xbatch" ]]; then
      plugin_type="${type:5}"
    elif [[ "x${type:0:8}" == "xrealtime" ]]; then
      plugin_type="${type:8}"
      
    # FIXME: this type "postaction" is going away
    elif [[ "x${type}" == "xpostaction" ]]; then
      plugin_type="post-run-plugin"
    # END FIXME
    
    # FIXME: these types "prerun" and  "postrun" are currently not used
#     elif [[ "x${type}" == "xprerun" ]]; then
#       plugin_type="pre-run"
#     elif [[ "x${type}" == "xpostrun" ]]; then
#       plugin_type="post-run"
    # END FIXME
    elif [[ "x${type}" == "xaction" ]]; then
      plugin_type="action"
    else
      # assume of type transform; to be copied to both batch and realtime
      plugin_type="transform"
    fi
  fi
  
  local target_dir="${plugin_type}s"

  echo ${NON_TRANSFORM_TYPES} | grep -q ${plugin_type}
  if [ "$?" == "0" ]; then
    local target_dir="${plugin_type}s"
  else
    local target_dir="transforms"
  fi

  local target="${BASE_TARGET}/${target_dir}/${target_file_name}"

  # Create display names for log output
  local fifty_spaces="                                                  "
  local display_source_file_name="${source_file_name}${fifty_spaces}"
  display_source_file_name="${display_source_file_name:0:50}"
  local display_source_dir="${source_dir}${fifty_spaces}"
  display_source_dir="${display_source_dir:0:25}"

  if curl --output /dev/null --silent --head --fail "${source_url}"; then
    echo "Downloading ${display_source_file_name} from ${display_source_dir} to ${target_dir}/${target_file_name}"
    #     Downloading Cassandra-batchsink.md from cassandra-plugins to batch/sink/cassandra.md
    curl --silent ${source_url} --output ${target}
    if [ -e "${target}" ]; then
      # If file does not begin with a "#" character, append "# title\n" to start
      local first=$(head -1 ${target})
      if [[ "x${first:0:2}" != "x# " ]]; then
        local m="Markdown file missing initial title: ${source_file_name}: ${source_name} ${type_capital}"
        echo_red_bold "${m}"
        set_message "${m}"
#         echo "# ${source_name} ${type_capital}${DOUBLE_RETURN_STRING}$(cat ${target})" > ${target}
        echo "# ${source_name}${DOUBLE_RETURN_STRING}$(cat ${target})" > ${target}
      else
        # Remove title suffixes
        tail -n +2 "${target}" > "${target}.tmp" && mv "${target}.tmp" "${target}"
        first="${first%"${first##*[![:space:]]}"}"
        first=${first% Batch Sink}
        first=${first% Batch Source}
        first=${first% Post Action}
        first=${first% Real-time Sink}
        first=${first% Real-time Source}
        first=${first% Source}
        first=${first% Transform}
        echo "${first}${DOUBLE_RETURN_STRING}$(cat ${target})" > ${target}
      fi
      if [[ "x${append_file}" != "x" ]]; then
        echo "  Appending ${append_file} to ${target_file_name}"
        cat ${BASE_TARGET}/${append_file} >> ${target}
      fi
      echo "${DOUBLE_RETURN_STRING}${RULE}- ${PLUGIN_TYPE_STRING} ${type}${DOUBLE_RETURN_STRING}- ${VERSION_STRING} ${HYDRATOR_VERSION}" >> ${target}
    else
      local m="File does not exist for ${target}"
      echo_red_bold "From ${source_url}"
      echo_red_bold "${m}"
      set_message "${m}"
    fi
  else
    local m="URL does not exist: ${source_url}"
    echo_red_bold "${m}"
    set_message "${m}"
  fi
}

function extract_table() {
  # Extracts a table written in Markdown from a file so it can be reused in reST files
{'fraction_non_alphanumeric': 0.16076656907106734, 'fraction_numerical': 0.006121905775885015, 'mean_word_length': 3.890238611713666, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2344340', 'n_tokens_mistral': 3736, 'n_tokens_neox': 3671, 'n_words': 930}