<commit_before># @@@ START COPYRIGHT @@@
#
# (C) Copyright 2014-2015 Hewlett-Packard Development Company, L.P.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
# @@@ END COPYRIGHT @@@

# Configuration file for the Trafodion Scanner
#
# 1. Specification format for Configuration Gathering is:
#    <configuration item> ::: <description> ::: <command>
#
#    <configuration item> must be set to CONFIGURATION_DUMP.
#    <description> is an English description for the configuration item.
#    <command> is the configuration gathering command to execute. This command will be executed
#              on each node specified by the trafodion_scanner --nodes parameter. Special variables 
#              can be specified in the command string within <<<>>>. The Trafodion Scanner will
#              replace each placeholder by its corresponding value, as determined at runtime.
#              Currently the only available variable is <<<hadoop_distribution>>>, which tells  
#              the type of Hadoop distribution installed: 'ambari' or 'cloudera'. The command
#              string can span multiple lines. However, pipelines within the commnand string
#              cannot be split accross multiple lines.
#
# 2. Specification format for Configuration Checks is:
#    <configuration item> ::: <description> ::: <severity level> ::: <expected value> ::: <comparison operator> ::: <command>
#
#    <configuration item> is a short identifier for the configuration item. The identifier CONFIGURATION_DUMP
#                         is reserved for Configuration Gathering.
#    <description> is an English description for the configuration item.
#    <severity level> is the severity level for the configuration item. The possible values are 'error' and 'warning',
#                     which correspond to required and recommended configuration items, respectively.
#    <expected value> is the expected value for the configuration item.
#    <comparison operator> is the operator to use for comparison of the <expected value> and "<command> output". 
#                          There are two types of operators: for string comparison and integer comparison. 
#                          The available operators for string comparison are: str_lt, str_le, str_eq, str_ne, str_ge, str_gt.
#                          The available operators for integer comparison are: int_lt, int_le, int_eq, int_ne, int_ge, int_gt.
#                          Here is the meaning for each operator: lt means "less than", le means "less than or equal", 
#                          eq means "equal", ne means "not equal", ge means "greater than or equal", gt means "greater than".
#                          The operator will be applied as follows: <expected value> <comparison operator> "<command> output",
#                          for example: <expected value> int_le "<command> output". The comparison will be done only once, on
#                          the entire "<command> output", even if it contains multiple values/lines.
#    <command> is the configuration check command to execute. This command will be executed on each node specified by
#              the trafodion_scanner --nodes parameter. Special variables can be specified in the command string within <<<>>>.
#              The Trafodion Scanner will replace each placeholder by its corresponding value, as determined at runtime.
#              Currently the only available variable is <<<hadoop_distribution>>>, which tells the type of 
#              Hadoop distribution installed: 'ambari' or 'cloudera'. The command string can span multiple lines. 
#              However, pipelines within the commnand string cannot be split accross multiple lines.
#
# 3. A line with comments must start with a '#' character.
#

### Configuration Gathering ###

CONFIGURATION_DUMP ::: Number of cores ::: grep -c '^processor' /proc/cpuinfo

CONFIGURATION_DUMP ::: GB of total physical memory ::: grep MemTotal /proc/meminfo | awk '{printf ("%.0f",$2/(1024*1024))}'

CONFIGURATION_DUMP ::: Linux distribution ::: lsb_release -a

CONFIGURATION_DUMP ::: User limit settings ::: ulimit -a

CONFIGURATION_DUMP ::: System information ::: uname -a

CONFIGURATION_DUMP ::: Network configuration ::: /sbin/ip addr show

CONFIGURATION_DUMP ::: Available filesystems ::: df -h

CONFIGURATION_DUMP ::: Installed RPMs ::: rpm -qa | sort

CONFIGURATION_DUMP ::: Hadoop distribution ::: echo <<<hadoop_distribution>>>

CONFIGURATION_DUMP ::: Hadoop version ::: hadoop version 

CONFIGURATION_DUMP ::: HBase version ::: hbase version

CONFIGURATION_DUMP ::: HBase Java version :::
hbaseJavaPath=$(ps -C java --no-headers -o cmd | grep -m 1 'org.apache.hadoop.hbase' | awk '{print $1}')
$hbaseJavaPath -version

CONFIGURATION_DUMP ::: Default Java version ::: java -version

### Configuration Checks ###

HardwarePlatform ::: Supported hardware platform ::: error ::: x86_64 ::: str_eq ::: uname -i

MinNumCores ::: Minimum number of cores ::: warning ::: 2 ::: int_le ::: grep -c '^processor' /proc/cpuinfo

MinGBTotalPhysicalMemory ::: Minimum GB of total physical memory ::: warning ::: 16 ::: int_le ::: 
grep MemTotal /proc/meminfo | awk '{printf ("%.0f",$2/(1024*1024))}'

LinuxDistro ::: Supported Linux distribution ::: error ::: 0 ::: int_ne ::: lsb_release -a | grep -c -e 'Red' -e 'CentOS'

ValidHostnameFQDNConfig ::: Valid hostname FQDN configuration ::: warning ::: PASS ::: str_eq ::: 
if [[ "$(hostname -f)" == "$(hostname -s)" ]]; then 
  echo FAIL
elif [[ "$(hostname -f | grep -c '\.')" != "1" ]]; then 
  echo FAIL
else 
  echo PASS
fi

ntpdServiceRunning ::: ntpd service is running ::: error ::: 0 ::: int_ne ::: /sbin/service ntpd status | grep -c 'is running'

iptablesFirewallNotEnabled ::: iptables firewall is not enabled ::: error ::: notEnabled ::: str_eq ::: 
stat=$(sudo -n /sbin/service iptables status 2>&1)
if echo "$stat" | grep -q -e 'is not running' -e 'unrecognized service'; then 
  echo notEnabled
elif echo "$stat" | grep -q '^[0-9]'; then 
  echo enabled
else 
  echo notEnabled
fi

HBaseVersion ::: Supported HBase version ::: error ::: 0 ::: int_ne ::: 
hbase version 2>&1 | grep -c -e 'HBase 0.98.0.2' -e 'HBase 0.98.1'

HBaseJavaVersion ::: Supported HBase Java version ::: error ::: 0 ::: int_ne ::: 
ps -C java --no-headers -o cmd | grep -m 1 'org.apache.hadoop.hbase' | awk '{print $1}' | grep -c '1.7.'

unixODBCVersion ::: Recommended unixODBC version ::: warning ::: 3 ::: int_le ::: 
ver=$(rpm -qa 'unixODBC' | cut -d '.' -f 2 | sort -n | tail -n 1)
if [[ -z $ver ]]; then 
  echo 0
else 
  echo $ver
fi

HyperthreadingEnabled ::: Hyperthreading is enabled, if capability exists ::: warning ::: 1 ::: int_ne :::
if grep '^flags' /proc/cpuinfo | grep -iq 'HT'; then 
  grep -E '^(cpu cores|siblings)' /proc/cpuinfo | cut -d ':' -f 2 | uniq | wc -l
else 
  echo 0
fi<commit_msg>scanner improvements<commit_after># @@@ START COPYRIGHT @@@
#
# (C) Copyright 2014-2015 Hewlett-Packard Development Company, L.P.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
# @@@ END COPYRIGHT @@@

# Configuration file for the Trafodion Scanner
#
# 1. Specification format for Configuration Gathering is:
#    <configuration item> ::: <description> ::: <command>
#
#    <configuration item> must be set to CONFIGURATION_DUMP.
#    <description> is an English description for the configuration item.
#    <command> is the configuration gathering command to execute. This command will be executed
#              on each node specified by the trafodion_scanner --nodes parameter. Special variables 
#              can be specified in the command string within <<<>>>. The Trafodion Scanner will
#              replace each placeholder by its corresponding value, as determined at runtime.
#              Currently the only available variable is <<<hadoop_distribution>>>, which tells  
#              the type of Hadoop distribution installed: 'ambari' or 'cloudera'. The command
#              string can span multiple lines. However, pipelines within the commnand string
#              cannot be split accross multiple lines.
#
# 2. Specification format for Configuration Checks is:
#    <configuration item> ::: <description> ::: <severity level> ::: <expected value> ::: <comparison operator> ::: <command>
#
#    <configuration item> is a short identifier for the configuration item. The identifier CONFIGURATION_DUMP
#                         is reserved for Configuration Gathering.
#    <description> is an English description for the configuration item.
#    <severity level> is the severity level for the configuration item. The possible values are 'error' and 'warning',
#                     which correspond to required and recommended configuration items, respectively.
#    <expected value> is the expected value for the configuration item.
#    <comparison operator> is the operator to use for comparison of the <expected value> and "<command> output". 
#                          There are two types of operators: for string comparison and integer comparison. 
#                          The available operators for string comparison are: str_lt, str_le, str_eq, str_ne, str_ge, str_gt.
#                          The available operators for integer comparison are: int_lt, int_le, int_eq, int_ne, int_ge, int_gt.
#                          Here is the meaning for each operator: lt means "less than", le means "less than or equal", 
#                          eq means "equal", ne means "not equal", ge means "greater than or equal", gt means "greater than".
#                          The operator will be applied as follows: <expected value> <comparison operator> "<command> output",
#                          for example: <expected value> int_le "<command> output". The comparison will be done only once, on
#                          the entire "<command> output", even if it contains multiple values/lines.
#    <command> is the configuration check command to execute. This command will be executed on each node specified by
#              the trafodion_scanner --nodes parameter. Special variables can be specified in the command string within <<<>>>.
#              The Trafodion Scanner will replace each placeholder by its corresponding value, as determined at runtime.
#              Currently the only available variable is <<<hadoop_distribution>>>, which tells the type of 
#              Hadoop distribution installed: 'ambari' or 'cloudera'. The command string can span multiple lines. 
#              However, pipelines within the commnand string cannot be split accross multiple lines.
#
# 3. A line with comments must start with a '#' character.
#

### Configuration Gathering ###

CONFIGURATION_DUMP ::: Number of cores ::: grep -c '^processor' /proc/cpuinfo

CONFIGURATION_DUMP ::: GB of total physical memory ::: grep MemTotal /proc/meminfo | awk '{printf ("%.0f",$2/(1024*1024))}'

CONFIGURATION_DUMP ::: Linux distribution ::: lsb_release -a

CONFIGURATION_DUMP ::: User limit settings ::: ulimit -a

CONFIGURATION_DUMP ::: System information ::: uname -a

CONFIGURATION_DUMP ::: Network configuration ::: /sbin/ip addr show

CONFIGURATION_DUMP ::: Available filesystems ::: df -h

CONFIGURATION_DUMP ::: Installed RPMs ::: rpm -qa | sort

CONFIGURATION_DUMP ::: Hadoop distribution ::: echo <<<hadoop_distribution>>>

CONFIGURATION_DUMP ::: Hadoop version ::: hadoop version 

CONFIGURATION_DUMP ::: HBase version ::: hbase version

CONFIGURATION_DUMP ::: HBase Java version :::
hbaseJavaPath=$(ps -C java --no-headers -o cmd | grep -m 1 'org.apache.hadoop.hbase' | awk '{print $1}')
$hbaseJavaPath -version

CONFIGURATION_DUMP ::: Default Java version ::: java -version

### Configuration Checks ###

HardwarePlatform ::: Supported hardware platform ::: error ::: x86_64 ::: str_eq ::: uname -i

MinNumCores ::: Minimum number of cores ::: warning ::: 2 ::: int_le ::: grep -c '^processor' /proc/cpuinfo

MinGBTotalPhysicalMemory ::: Minimum GB of total physical memory ::: warning ::: 16 ::: int_le ::: 
grep MemTotal /proc/meminfo | awk '{printf ("%.0f",$2/(1024*1024))}'

LinuxDistro ::: Supported Linux distribution ::: error ::: 0 ::: int_ne ::: lsb_release -a | grep -c -e 'Red' -e 'CentOS'

ValidHostnameFQDNConfig ::: Valid hostname FQDN configuration ::: warning ::: PASS ::: str_eq ::: 
if [[ "$(hostname -f)" == "$(hostname -s)" ]]; then 
  echo FAIL
elif [[ "$(hostname -f | grep -c '\.')" != "1" ]]; then 
  echo FAIL
else 
  echo PASS
fi

ntpdServiceRunning ::: ntpd service is running ::: error ::: 0 ::: int_ne ::: /sbin/service ntpd status | grep -c 'is running'

iptablesFirewallNotEnabled ::: iptables firewall is not enabled ::: error ::: notEnabled ::: str_eq ::: 
stat=$(sudo -n /sbin/service iptables status 2>&1)
if echo "$stat" | grep -q -e 'is not running' -e 'unrecognized service'; then 
  echo notEnabled
elif echo "$stat" | grep -q '^[0-9]'; then 
  echo enabled
else 
  echo notEnabled
fi

HBaseVersion ::: Supported HBase version ::: error ::: 0 ::: int_ne ::: 
hbase version 2>&1 | grep -c -e 'HBase 0.98.0.2' -e 'HBase 0.98.1'

HBaseJavaVersion ::: Supported HBase Java version ::: error ::: 0 ::: int_ne ::: 
ps -C java --no-headers -o cmd | grep -m 1 'org.apache.hadoop.hbase' | awk '{print $1}' | grep -c '1.7.'

HyperthreadingEnabled ::: Hyperthreading is enabled, if capability exists ::: warning ::: 1 ::: int_ne :::
if grep '^flags' /proc/cpuinfo | grep -iq 'HT'; then 
  grep -E '^(cpu cores|siblings)' /proc/cpuinfo | cut -d ':' -f 2 | uniq | wc -l
else 
  echo 0
fi

YumAndInternetConnection ::: Yum and Internet connection ::: warning ::: PASS ::: str_eq ::: 
which yumdownloader &>/dev/null
if [[ "$?" == "0" ]]; then 
  mkdir /tmp/yum-check-temp-dir-$$ &>/dev/null
  cmdout=$(yumdownloader --destdir /tmp/yum-check-temp-dir-$$ 'pdsh' 2>&1)
  if [[ "$?" == "0" ]]; then 
    echo PASS
  else
    echo "$cmdout"
  fi
  rm -rf /tmp/yum-check-temp-dir-$$ &>/dev/null
else 
  cmdout=$(wget --spider 'http://downloads.trafodion.org' 2>&1)
  if [[ "$?" == "0" ]]; then 
    echo PASS
  else
    echo "$cmdout"
  fi
fi
{'fraction_non_alphanumeric': 0.11663238904000531, 'fraction_numerical': 0.01015060041133152, 'mean_word_length': 3.3229136793805565, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1869082', 'n_tokens_mistral': 4503, 'n_tokens_neox': 4276, 'n_words': 1721}