<commit_before>#!/bin/bash

# Copyright 2020 The SQLFlow Authors. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -ex

if [ "$FIND_FASTED_MIRROR" == "true" ]; then
    # shellcheck disable=SC1091
    source find_fastest_resources.sh
    echo "Choose the fastest APT source ..."
    choose_fastest_apt_source
    echo "Choose the fastest PIP source ..."
    choose_fastest_pip_source
fi


echo "Install apt packages ..."
DOWNLOAD_TOOLS="curl axel unzip" # We need curl to check the running of Hive.
BUILD_ESSENTIAL="build-essential git"
PYTHON_DEV="python3-dev python3-pip" # Many pip packages require Python.h
JAVA_DEV="openjdk-8-jdk maven"
SHELL_LINTER="shellcheck"
YAML_LINTER="yamllint"
OPTIMIZE_SOLVER="glpk-utils" # required solver packages of Pyomo
# shellcheck disable=SC2086
apt-get -qq update && apt-get -qq install -y --no-install-recommends \
        $DOWNLOAD_TOOLS \
        $BUILD_ESSENTIAL \
        $PYTHON_DEV \
        $JAVA_DEV \
        $SHELL_LINTER \
        $YAML_LINTER \
        $OPTIMIZE_SOLVER \
        > /dev/null
rm -rf /var/lib/apt/lists/*
apt-get -qq clean -y


echo "Make Python 3 the the default ..."
ln -s /usr/bin/python3 /usr/local/bin/python
ln -s /usr/bin/pip3 /usr/local/bin/pip

echo "Upgrade pip and setuptools creates /usr/local/bin/pip ..."
# Update setuptools because
# https://github.com/red-hat-storage/ocs-ci/pull/971/files
python -m pip install --quiet --upgrade pip setuptools six


echo "Install pip packages ..."
PRE_COMMIT="pre-commit==1.18.3"
PY_TEST="pytest==5.3.0 pytest-cov"
GRPC_PACKAGES="grpcio==1.28.1 grpcio-tools==1.28.1"
JS_LINTER=jsbeautifier
PYTHON_LINTER="yapf isort<5,>=4.2.5 pylint>=2.5.3 flake8"
WHEEL="wheel"
# shellcheck disable=SC2086
python -m pip install --quiet \
    $WHEEL \
    $PRE_COMMIT \
    $PY_TEST \
    $JS_LINTER \
    $PYTHON_LINTER \
    $GRPC_PACKAGES
rm -rf "$HOME"/.cache/pip/*


echo "Install Go compiler ..."
GO_MIRROR_0="https://studygolang.com/dl/golang/go1.13.4.linux-amd64.tar.gz"
GO_MIRROR_1="https://dl.google.com/go/go1.13.4.linux-amd64.tar.gz"
axel --quiet --output go.tar.gz $GO_MIRROR_0 $GO_MIRROR_1
tar -C /usr/local -xzf go.tar.gz
rm go.tar.gz
export GOPATH="/root/go"
export PATH="/usr/local/go/bin:$GOPATH/bin:$PATH"


echo "Install goyacc, protoc-gen-go, linters, etc. ..."
# Set the env system-wide for later usage, e.g. build source
go env -w GO111MODULE=on<commit_msg>add kits for developer (#3071)<commit_after>#!/bin/bash

# Copyright 2020 The SQLFlow Authors. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -ex

if [ "$FIND_FASTED_MIRROR" == "true" ]; then
    # shellcheck disable=SC1091
    source find_fastest_resources.sh
    echo "Choose the fastest APT source ..."
    choose_fastest_apt_source
    echo "Choose the fastest PIP source ..."
    choose_fastest_pip_source
fi


echo "Install apt packages ..."
DOWNLOAD_TOOLS="curl axel unzip" # We need curl to check the running of Hive.
BUILD_ESSENTIAL="build-essential git"
PYTHON_DEV="python3-dev python3-pip" # Many pip packages require Python.h
JAVA_DEV="openjdk-8-jdk maven"
SHELL_LINTER="shellcheck"
YAML_LINTER="yamllint"
OPTIMIZE_SOLVER="glpk-utils" # required solver packages of Pyomo
# TODO(weiguoz) let's remove ${SASL_LIB} and ${DEVELOPER_KITS} when the debug is completed.
SASL_LIB="libsasl2-dev"
DEVELOPER_KITS="tmux vim protobuf-compiler libsasl2-dev"
# shellcheck disable=SC2086
apt-get -qq update && apt-get -qq install -y --no-install-recommends \
        $DOWNLOAD_TOOLS \
        $BUILD_ESSENTIAL \
        $PYTHON_DEV \
        $JAVA_DEV \
        $SHELL_LINTER \
        $YAML_LINTER \
        $OPTIMIZE_SOLVER \
        $SASL_LIB \
        $DEVELOPER_KITS \
        > /dev/null
rm -rf /var/lib/apt/lists/*
apt-get -qq clean -y


echo "Make Python 3 the the default ..."
ln -s /usr/bin/python3 /usr/local/bin/python
ln -s /usr/bin/pip3 /usr/local/bin/pip

echo "Upgrade pip and setuptools creates /usr/local/bin/pip ..."
# Update setuptools because
# https://github.com/red-hat-storage/ocs-ci/pull/971/files
python -m pip install --quiet --upgrade pip setuptools six


echo "Install pip packages ..."
PRE_COMMIT="pre-commit==1.18.3"
PY_TEST="pytest==5.3.0 pytest-cov"
GRPC_PACKAGES="grpcio==1.28.1 grpcio-tools==1.28.1"
JS_LINTER=jsbeautifier
PYTHON_LINTER="yapf isort<5,>=4.2.5 pylint>=2.5.3 flake8"
WHEEL="wheel"
THRIFT_SASL="thrift_sasl==0.4.2"
# shellcheck disable=SC2086
python -m pip install --quiet \
    $WHEEL \
    $PRE_COMMIT \
    $PY_TEST \
    $JS_LINTER \
    $PYTHON_LINTER \
    $GRPC_PACKAGES \
    $THRIFT_SASL
rm -rf "$HOME"/.cache/pip/*


echo "Install Go compiler ..."
GO_MIRROR_0="https://studygolang.com/dl/golang/go1.13.4.linux-amd64.tar.gz"
GO_MIRROR_1="https://dl.google.com/go/go1.13.4.linux-amd64.tar.gz"
axel --quiet --output go.tar.gz $GO_MIRROR_0 $GO_MIRROR_1
tar -C /usr/local -xzf go.tar.gz
rm go.tar.gz
export GOPATH="/root/go"
export PATH="/usr/local/go/bin:$GOPATH/bin:$PATH"


echo "Install goyacc, protoc-gen-go, linters, etc. ..."
# Set the env system-wide for later usage, e.g. build source
go env -w GO111MODULE=on
{'fraction_non_alphanumeric': 0.1242603550295858, 'fraction_numerical': 0.024819197896120974, 'mean_word_length': 3.487463126843658, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5071262', 'n_tokens_mistral': 2351, 'n_tokens_neox': 2193, 'n_words': 585}