<commit_before>#!/bin/bash
# Copyright 2020 The FedLearner Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


ACCESS_KEY_ID=$1
ACCESS_KEY_SECRET=$2
DB_PASSWORD=$3

REGION="cn-beijing"
ZONE_ID="cn-beijing-h"
GENERATER_NAME="fedlearnerwins"
VPC_CIDR_BLOCK="192.168.0.0/16"

function echo_exit {
    echo $1
    exit 1
}

function echo_log {
    msg=$1
    echo $msg
    echo $msg >> install.log
}

function json2yaml {
    python -c 'import json; open("config", "w").write(json.load(open("./tmp","r"))["config"]);'
}

function install_cli {
    # Download kubectl
    kubectl help >/dev/null 2>&1
    if [ $? -ne 0 ]
    then
        echo_log "Download kubectl."
        curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl
        mv kubectl /usr/local/bin/
        chmod 755 /usr/local/bin/kubectl
    fi

    # Download helm
    helm version | grep Version:\"v3 >/dev/null 2>&1
    if [ $? -ne 0 ]
    then
        echo_log "Download helm."
        curl -LO https://get.helm.sh/helm-v3.2.3-darwin-amd64.tar.gz
        tar -zxvf helm-v3.2.3-darwin-amd64.tar.gz
        mv darwin-amd64/helm /usr/local/bin/
        chmod 755 /usr/local/bin/helm
        rm -rf darwin-amd64 helm-v3.2.3-darwin-amd64.tar.gz
    fi

    # Download aliyun cli
    aliyun version >/dev/null 2>&1
    if [ $? -ne 0 ]
    then
        echo_log "Download aliyun cli."
        curl -LO https://aliyuncli.alicdn.com/aliyun-cli-macosx-3.0.32-amd64.tgz
        tar -zxvf aliyun-cli-macosx-3.0.32-amd64.tgz
        mv aliyun /usr/local/bin
        chmod 755 /usr/local/bin/aliyun
        rm -rf aliyun-cli-macosx-3.0.32-amd64.tgz
    fi

    # Configure aliyun cli
    aliyun auto-completion
    aliyun configure set --profile akProfile --region $REGION --access-key-id $ACCESS_KEY_ID --access-key-secret $ACCESS_KEY_SECRET --language en
    if [ $? -ne 0 ]
    then
        echo_exit "Failed to initiate aliyun cli."
    fi
}

function create_role {
    ROLE_NAME=$1
    POLICY_NAME=$2
    SERVICE=$3
    aliyun ram CreateRole --RoleName $ROLE_NAME --AssumeRolePolicyDocument "{\"Statement\": [{\"Action\": \"sts:AssumeRole\",\"Effect\": \"Allow\",\"Principal\": {\"Service\": [\"$SERVICE\"]}}],\"Version\": \"1\"}" >/dev/null 2>&1
    aliyun ram AttachPolicyToRole --RoleName $ROLE_NAME --PolicyType System --PolicyName $POLICY_NAME >/dev/null 2>&1
}

function init_policy {
    create_role AliyunNASDefaultRole             AliyunNASRolePolicy                    nas.aliyuncs.com
    create_role AliyunCSClusterRole              AliyunCSClusterRolePolicy              cs.aliyuncs.com
    create_role AliyunCSDefaultRole              AliyunCSDefaultRolePolicy              cs.aliyuncs.com
    create_role AliyunCSKubernetesAuditRole      AliyunCSKubernetesAuditRolePolicy      cs.aliyuncs.com
    create_role AliyunCSManagedArmsRole          AliyunCSManagedArmsRolePolicy          cs.aliyuncs.com
    create_role AliyunCSManagedCmsRole           AliyunCSManagedCmsRolePolicy           cs.aliyuncs.com
    create_role AliyunCSManagedCsiRole           AliyunCSManagedCsiRolePolicy           cs.aliyuncs.com
    create_role AliyunCSManagedKubernetesRole    AliyunCSManagedKubernetesRolePolicy    cs.aliyuncs.com
    create_role AliyunCSManagedLogRole           AliyunCSManagedLogRolePolicy           cs.aliyuncs.com
    create_role AliyunCSManagedNetworkRole       AliyunCSManagedNetworkRolePolicy       cs.aliyuncs.com
    create_role AliyunCSManagedVKRole            AliyunCSManagedVKRolePolicy            cs.aliyuncs.com
    create_role AliyunCSServerlessKubernetesRole AliyunCSServerlessKubernetesRolePolicy cs.aliyuncs.com
    create_role AliyunESSDefaultRole             AliyunESSRolePolicy                    cs.aliyuncs.com

}

function create_oss_bucket {
    aliyun oss mb oss://fedlearner --storage-class Standard >/dev/null 2>&1
}

function create_vpc {
    VPC_ID=`aliyun vpc DescribeVpcs --VpcName $GENERATER_NAME | grep VpcId | awk -F "\"" '{print $4}'`
    if [[ $VPC_ID == "vpc"* ]]
    then
        echo_log "Vpc $GENERATER_NAME already exists with vpc id $VPC_ID."
    else
        aliyun vpc CreateVpc --CidrBlock $VPC_CIDR_BLOCK --VpcName $GENERATER_NAME
        VPC_ID=`aliyun vpc DescribeVpcs --VpcName $GENERATER_NAME | grep VpcId | awk -F "\"" '{print $4}'`
        if [[ $VPC_ID == "vpc"* ]]
            then
            echo_log "Create vpc $GENERATER_NAME success with vpc id $VPC_ID."
        else
            echo_exit "Failed to create vpc"
        fi

        sleep 5
    fi
}

function create_vswitch {
    VSWITCH_ID=`aliyun vpc DescribeVSwitches --VSwitchName $GENERATER_NAME | grep VSwitchId | awk -F "\"" '{print $4}'`
    if [[ $VSWITCH_ID == "vsw"* ]]
    then
        echo_log "Vswitch $GENERATER_NAME already exists with vswitch id $VSWITCH_ID."
    else
        aliyun vpc CreateVSwitch --VpcId $VPC_ID --CidrBlock $VPC_CIDR_BLOCK --VSwitchName $GENERATER_NAME --ZoneId $ZONE_ID
        VSWITCH_ID=`aliyun vpc DescribeVSwitches --VSwitchName $GENERATER_NAME | grep VSwitchId | awk -F "\"" '{print $4}'`
        if [[ $VSWITCH_ID == "vsw"* ]]
        then
            echo_log "Create vswtich $GENERATER_NAME success with vswtich id $VSWITCH_ID."
        else
            echo_exit "Failed to create vswtich"
        fi

        sleep 5
    fi
}

function create_secret {
    KEYPAIR_NAME=`aliyun ecs DescribeKeyPairs --KeyPairName $GENERATER_NAME | grep KeyPairName | awk -F "\"" '{print $4}'`
    if [ -n "$KEYPAIR_NAME" ]
    then
        echo_log "Keypair $GENERATER_NAME already exists."
    else
        aliyun ecs CreateKeyPair --KeyPairName $GENERATER_NAME
        if [ $? -eq 0 ]
        then
            echo_log "Create keypair $GENERATER_NAME Success."
        else
            echo_exit "Failed to create keypair $GENERATER_NAME."
        fi
    fi
}

function create_k8s_cluster_config {
    rm -rf k8s.json

    echo_log "Generate the kubernetes cluster config file for aliyun."

    cat <<EOF >>k8s.json
{
    "name": "$GENERATER_NAME",
    "cluster_type": "ManagedKubernetes",
    "disable_rollback": true,
    "timeout_mins": 60,
    "kubernetes_version": "1.16.9-aliyun.1",
    "region_id": "$REGION",
    "snat_entry": true,
    "cloud_monitor_flags": false,
    "endpoint_public_access": true,
    "deletion_protection": false,
    "node_cidr_mask": "26",
    "proxy_mode": "ipvs",
    "tags": [],
    "addons": [
        {
            "name": "flannel"
        },
        {
            "name": "csi-plugin"
        },
        {
            "name": "csi-provisioner"
        },
        {
            "name": "nginx-ingress-controller",
            "disabled": true
        }
    ],
    "os_type": "Linux",
    "platform": "CentOS",
    "runtime": {
        "name": "docker",
        "version": "19.03.5"
    },
    "worker_instance_types": [
        "ecs.c6.3xlarge"
    ],
    "num_of_nodes": 3,
    "worker_system_disk_category": "cloud_efficiency",
    "worker_system_disk_size": 120,
    "worker_instance_charge_type": "PostPaid",
    "vpcid": "$VPC_ID",
    "container_cidr": "172.20.0.0/16",
    "service_cidr": "172.21.0.0/20",
    "vswitch_ids": [
        "$VSWITCH_ID"
    ],
    "key_pair": "$GENERATER_NAME",
    "cpu_policy": "none",
    "is_enterprise_security_group": true
}
EOF
}

function create_k8s {
    create_k8s_cluster_config

    CLUSTER_ID=`aliyun cs DescribeClusters | grep -A 1 name | grep -A 1 $GENERATER_NAME | grep cluster_id | awk -F "\"" '{print $4}'`
    if [ -n "$CLUSTER_ID" ]
    then
        echo_log "Kubernetes cluster $GENERATER_NAME already exists with id $CLUSTER_ID."
    else
        CLUSTER_ID=`aliyun cs POST /clusters --header "Content-Type=application/json" --body "$(cat ./k8s.json)" | grep cluster_id | awk -F "\"" '{print $4}'`
        if [ -n "$CLUSTER_ID" ]
        then
            echo_log "Kubernetes cluster $GENERATER_NAME create success with id $CLUSTER_ID."
            STATUS=`aliyun cs DescribeClusters | grep -A 5 name | grep -A 5 $GENERATER_NAME | grep state | awk -F "\"" '{print $4}'`
            while [ "$STATUS" != "running" ]
            do
                echo_log "Current kubernetes cluster status is $STATUS, loop wait until it's running."
                sleep 30
                STATUS=`aliyun cs DescribeClusters | grep -A 5 name | grep -A 5 $GENERATER_NAME | grep state | awk -F "\"" '{print $4}'`
            done
        else
            echo_exit "Failed to create k8s cluster $GENERATER_NAME."
        fi
    fi

    CLUSTER_ID=`aliyun cs DescribeClusters | grep -A 1 name | grep -A 1 $GENERATER_NAME | grep cluster_id | awk -F "\"" '{print $4}'`

    rm -rf tmp config
    echo_log "Creating config file in current dir, you can move it to ~/.kube/config."
    aliyun cs GET /k8s/$CLUSTER_ID/user_config > ./tmp
    json2yaml
    CURRENT_DIR=`pwd`
    export KUBECONFIG="$CURRENT_DIR/config"
    rm -rf tmp k8s.json
}

function create_db {
    DB_INSTANCE_ID=`aliyun rds DescribeDBInstances --VpcId $VPC_ID | grep \"DBInstanceId\" | awk -F "\"" '{print $4}'`
    if [ -n "$DB_INSTANCE_ID" ]
    then
        echo_log "Database already exists with id $DB_INSTANCE_ID."
    else
        aliyun rds CreateDBInstance --Engine MySQL --EngineVersion 8.0 --DBInstanceClass rds.mysql.t1.small --DBInstanceStorage 20  --SecurityIPList 0.0.0.0/0 --PayType Postpaid --DBInstanceNetType Intranet --RegionId $REGION --ZoneId $ZONE_ID --VPCId $VPC_ID --InstanceNetworkType VPC
        DB_INSTANCE_ID=`aliyun rds DescribeDBInstances --VpcId $VPC_ID | grep \"DBInstanceId\" | awk -F "\"" '{print $4}'`
        if [ -n "$DB_INSTANCE_ID" ]
        then
            echo_log "Create db instance success with instance id $DB_INSTANCE_ID."
            STATUS=`aliyun rds DescribeDBInstances --VpcId $VPC_ID | grep DBInstanceStatus | awk -F "\"" '{print $4}'`
            while [ "$STATUS" != "Running" ]
            do
                echo_log "Current db instance status is $STATUS, loop wait until it's running."
                sleep 30
                STATUS=`aliyun rds DescribeDBInstances --VpcId $VPC_ID | grep DBInstanceStatus | awk -F "\"" '{print $4}'`
            done
        else
            echo_exit "Failed to create db instance."
        fi
    fi

    # Create database account.
    DB_ACCOUNT=`aliyun rds DescribeAccounts --DBInstanceId $DB_INSTANCE_ID | grep fedlearner`
    if [ -z "$DB_ACCOUNT" ]
    then
        aliyun rds CreateAccount --AccountName fedlearner --AccountPassword $<PASSWORD> --AccountType Super --DBInstanceId $DB_INSTANCE_ID
        if [ $? -eq 0 ]
        then
            echo_log "Create db account fedlearner success."
            sleep 5<commit_msg>change the paytype to prepay (#295)<commit_after>#!/bin/bash
# Copyright 2020 The FedLearner Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


ACCESS_KEY_ID=$1
ACCESS_KEY_SECRET=$2
DB_PASSWORD=$3
BUCKET=$4

REGION="cn-beijing"
ZONE_ID="cn-beijing-h"
GENERATER_NAME="fedlearnerwins"
VPC_CIDR_BLOCK="192.168.0.0/16"

function echo_exit {
    echo $1
    exit 1
}

function echo_log {
    msg=$1
    echo $msg
    echo $msg >> install.log
}

function json2yaml {
    python -c 'import json; open("config", "w").write(json.load(open("./tmp","r"))["config"]);'
}

function install_cli {
    # Download kubectl
    kubectl help >/dev/null 2>&1
    if [ $? -ne 0 ]
    then
        echo_log "Download kubectl."
        curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl
        mv kubectl /usr/local/bin/
        chmod 755 /usr/local/bin/kubectl
    fi

    # Download helm
    helm version | grep Version:\"v3 >/dev/null 2>&1
    if [ $? -ne 0 ]
    then
        echo_log "Download helm."
        curl -LO https://get.helm.sh/helm-v3.2.3-darwin-amd64.tar.gz
        tar -zxvf helm-v3.2.3-darwin-amd64.tar.gz
        mv darwin-amd64/helm /usr/local/bin/
        chmod 755 /usr/local/bin/helm
        rm -rf darwin-amd64 helm-v3.2.3-darwin-amd64.tar.gz
    fi

    # Download aliyun cli
    aliyun version >/dev/null 2>&1
    if [ $? -ne 0 ]
    then
        echo_log "Download aliyun cli."
        curl -LO https://aliyuncli.alicdn.com/aliyun-cli-macosx-3.0.32-amd64.tgz
        tar -zxvf aliyun-cli-macosx-3.0.32-amd64.tgz
        mv aliyun /usr/local/bin
        chmod 755 /usr/local/bin/aliyun
        rm -rf aliyun-cli-macosx-3.0.32-amd64.tgz
    fi

    # Configure aliyun cli
    aliyun auto-completion
    aliyun configure set --profile akProfile --region $REGION --access-key-id $ACCESS_KEY_ID --access-key-secret $ACCESS_KEY_SECRET --language en
    if [ $? -ne 0 ]
    then
        echo_exit "Failed to initiate aliyun cli."
    fi
}

function create_role {
    ROLE_NAME=$1
    POLICY_NAME=$2
    SERVICE=$3
    aliyun ram CreateRole --RoleName $ROLE_NAME --AssumeRolePolicyDocument "{\"Statement\": [{\"Action\": \"sts:AssumeRole\",\"Effect\": \"Allow\",\"Principal\": {\"Service\": [\"$SERVICE\"]}}],\"Version\": \"1\"}" >/dev/null 2>&1
    aliyun ram AttachPolicyToRole --RoleName $ROLE_NAME --PolicyType System --PolicyName $POLICY_NAME >/dev/null 2>&1
}

function init_policy {
    create_role AliyunNASDefaultRole             AliyunNASRolePolicy                    nas.aliyuncs.com
    create_role AliyunCSClusterRole              AliyunCSClusterRolePolicy              cs.aliyuncs.com
    create_role AliyunCSDefaultRole              AliyunCSDefaultRolePolicy              cs.aliyuncs.com
    create_role AliyunCSKubernetesAuditRole      AliyunCSKubernetesAuditRolePolicy      cs.aliyuncs.com
    create_role AliyunCSManagedArmsRole          AliyunCSManagedArmsRolePolicy          cs.aliyuncs.com
    create_role AliyunCSManagedCmsRole           AliyunCSManagedCmsRolePolicy           cs.aliyuncs.com
    create_role AliyunCSManagedCsiRole           AliyunCSManagedCsiRolePolicy           cs.aliyuncs.com
    create_role AliyunCSManagedKubernetesRole    AliyunCSManagedKubernetesRolePolicy    cs.aliyuncs.com
    create_role AliyunCSManagedLogRole           AliyunCSManagedLogRolePolicy           cs.aliyuncs.com
    create_role AliyunCSManagedNetworkRole       AliyunCSManagedNetworkRolePolicy       cs.aliyuncs.com
    create_role AliyunCSManagedVKRole            AliyunCSManagedVKRolePolicy            cs.aliyuncs.com
    create_role AliyunCSServerlessKubernetesRole AliyunCSServerlessKubernetesRolePolicy cs.aliyuncs.com
    create_role AliyunESSDefaultRole             AliyunESSRolePolicy                    cs.aliyuncs.com

}

function create_oss_bucket {
    aliyun oss mb oss://$BUCKET --storage-class Standard >/dev/null 2>&1
}

function create_vpc {
    VPC_ID=`aliyun vpc DescribeVpcs --VpcName $GENERATER_NAME | grep VpcId | awk -F "\"" '{print $4}'`
    if [[ $VPC_ID == "vpc"* ]]
    then
        echo_log "Vpc $GENERATER_NAME already exists with vpc id $VPC_ID."
    else
        aliyun vpc CreateVpc --CidrBlock $VPC_CIDR_BLOCK --VpcName $GENERATER_NAME
        VPC_ID=`aliyun vpc DescribeVpcs --VpcName $GENERATER_NAME | grep VpcId | awk -F "\"" '{print $4}'`
        if [[ $VPC_ID == "vpc"* ]]
            then
            echo_log "Create vpc $GENERATER_NAME success with vpc id $VPC_ID."
        else
            echo_exit "Failed to create vpc"
        fi

        sleep 5
    fi
}

function create_vswitch {
    VSWITCH_ID=`aliyun vpc DescribeVSwitches --VSwitchName $GENERATER_NAME | grep VSwitchId | awk -F "\"" '{print $4}'`
    if [[ $VSWITCH_ID == "vsw"* ]]
    then
        echo_log "Vswitch $GENERATER_NAME already exists with vswitch id $VSWITCH_ID."
    else
        aliyun vpc CreateVSwitch --VpcId $VPC_ID --CidrBlock $VPC_CIDR_BLOCK --VSwitchName $GENERATER_NAME --ZoneId $ZONE_ID
        VSWITCH_ID=`aliyun vpc DescribeVSwitches --VSwitchName $GENERATER_NAME | grep VSwitchId | awk -F "\"" '{print $4}'`
        if [[ $VSWITCH_ID == "vsw"* ]]
        then
            echo_log "Create vswtich $GENERATER_NAME success with vswtich id $VSWITCH_ID."
        else
            echo_exit "Failed to create vswtich"
        fi

        sleep 5
    fi
}

function create_secret {
    KEYPAIR_NAME=`aliyun ecs DescribeKeyPairs --KeyPairName $GENERATER_NAME | grep KeyPairName | awk -F "\"" '{print $4}'`
    if [ -n "$KEYPAIR_NAME" ]
    then
        echo_log "Keypair $GENERATER_NAME already exists."
    else
        aliyun ecs CreateKeyPair --KeyPairName $GENERATER_NAME
        if [ $? -eq 0 ]
        then
            echo_log "Create keypair $GENERATER_NAME Success."
        else
            echo_exit "Failed to create keypair $GENERATER_NAME."
        fi
    fi
}

function create_k8s_cluster_config {
    rm -rf k8s.json

    echo_log "Generate the kubernetes cluster config file for aliyun."

    cat <<EOF >>k8s.json
{
    "name": "$GENERATER_NAME",
    "cluster_type": "ManagedKubernetes",
    "disable_rollback": true,
    "timeout_mins": 60,
    "kubernetes_version": "1.16.9-aliyun.1",
    "region_id": "$REGION",
    "snat_entry": true,
    "cloud_monitor_flags": false,
    "endpoint_public_access": true,
    "deletion_protection": false,
    "node_cidr_mask": "26",
    "proxy_mode": "ipvs",
    "tags": [],
    "addons": [
        {
            "name": "flannel"
        },
        {
            "name": "csi-plugin"
        },
        {
            "name": "csi-provisioner"
        },
        {
            "name": "nginx-ingress-controller",
            "disabled": true
        }
    ],
    "os_type": "Linux",
    "platform": "CentOS",
    "runtime": {
        "name": "docker",
        "version": "19.03.5"
    },
    "worker_instance_types": [
        "ecs.c6.3xlarge"
    ],
    "num_of_nodes": 3,
    "worker_system_disk_category": "cloud_efficiency",
    "worker_system_disk_size": 120,
    "worker_instance_charge_type": "PrePaid",
    "worker_period_unit": "Month",
    "worker_period": 1,
    "worker_auto_renew": true,
    "worker_auto_renew_period": 1,
    "vpcid": "$VPC_ID",
    "container_cidr": "172.20.0.0/16",
    "service_cidr": "172.21.0.0/20",
    "vswitch_ids": [
        "$VSWITCH_ID"
    ],
    "key_pair": "$GENERATER_NAME",
    "cpu_policy": "none",
    "is_enterprise_security_group": true
}
EOF
}

function create_k8s {
    create_k8s_cluster_config

    CLUSTER_ID=`aliyun cs DescribeClusters | grep -A 1 name | grep -A 1 $GENERATER_NAME | grep cluster_id | awk -F "\"" '{print $4}'`
    if [ -n "$CLUSTER_ID" ]
    then
        echo_log "Kubernetes cluster $GENERATER_NAME already exists with id $CLUSTER_ID."
    else
        CLUSTER_ID=`aliyun cs POST /clusters --header "Content-Type=application/json" --body "$(cat ./k8s.json)" | grep cluster_id | awk -F "\"" '{print $4}'`
        if [ -n "$CLUSTER_ID" ]
        then
            echo_log "Kubernetes cluster $GENERATER_NAME create success with id $CLUSTER_ID."
            STATUS=`aliyun cs DescribeClusters | grep -A 5 name | grep -A 5 $GENERATER_NAME | grep state | awk -F "\"" '{print $4}'`
            while [ "$STATUS" != "running" ]
            do
                echo_log "Current kubernetes cluster status is $STATUS, loop wait until it's running."
                sleep 30
                STATUS=`aliyun cs DescribeClusters | grep -A 5 name | grep -A 5 $GENERATER_NAME | grep state | awk -F "\"" '{print $4}'`
            done
        else
            echo_exit "Failed to create k8s cluster $GENERATER_NAME."
        fi
    fi

    CLUSTER_ID=`aliyun cs DescribeClusters | grep -A 1 name | grep -A 1 $GENERATER_NAME | grep cluster_id | awk -F "\"" '{print $4}'`

    rm -rf tmp config
    echo_log "Creating config file in current dir, you can move it to ~/.kube/config."
    aliyun cs GET /k8s/$CLUSTER_ID/user_config > ./tmp
    json2yaml
    CURRENT_DIR=`pwd`
    export KUBECONFIG="$CURRENT_DIR/config"
    rm -rf tmp k8s.json
}

function create_db {
    DB_INSTANCE_ID=`aliyun rds DescribeDBInstances --VpcId $VPC_ID | grep \"DBInstanceId\" | awk -F "\"" '{print $4}' | head -1`
    if [ -n "$DB_INSTANCE_ID" ]
    then
        echo_log "Database already exists with id $DB_INSTANCE_ID."
    else
        aliyun rds CreateDBInstance --Engine MySQL --EngineVersion 8.0 --DBInstanceClass rds.mysql.t1.small --DBInstanceStorage 20  --SecurityIPList 0.0.0.0/0 --DBInstanceNetType Intranet --RegionId $REGION --ZoneId $ZONE_ID --VPCId $VPC_ID --InstanceNetworkType VPC --PayType Prepaid --UsedTime 1 --Period Month --AutoRenew true
        DB_INSTANCE_ID=`aliyun rds DescribeDBInstances --VpcId $VPC_ID | grep \"DBInstanceId\" | awk -F "\"" '{print $4}'`
        if [ -n "$DB_INSTANCE_ID" ]
        then
            echo_log "Create db instance success with instance id $DB_INSTANCE_ID."
            STATUS=`aliyun rds DescribeDBInstances --VpcId $VPC_ID | grep DBInstanceStatus | awk -F "\"" '{print $4}'`
            while [ "$STATUS" != "Running" ]
            do
                echo_log "Current db instance status is $STATUS, loop wait until it's running."
                sleep 30
                STATUS=`aliyun rds DescribeDBInstances --VpcId $VPC_ID | grep DBInstanceStatus | awk -F "\"" '{print $4}'`
            done
        else
            echo_exit "Failed to create db instance."
        fi
    fi

    # Create database account.
    DB_ACCOUNT=`aliyun rds DescribeAccounts --DBInstanceId $DB_INSTANCE_ID | grep fedlearner`
    if [ -z "$DB_ACCOUNT" ]
    then
        aliyun rds CreateAccount --AccountName fedlearner --AccountPassword $DB_PASSWORD --AccountType Super --DBInstanceId $DB_INSTANCE_ID
        if [ $? -eq 0 ]
        then
            echo_log "Create db account fedlearner success."
            sleep 5
{'fraction_non_alphanumeric': 0.11188066062866275, 'fraction_numerical': 0.01695968744450364, 'mean_word_length': 2.4982140083863955, 'pattern_counts': {'":': 88, '<': 8, '<?xml version=': 0, '>': 38, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4310397', 'n_tokens_mistral': 7971, 'n_tokens_neox': 7599, 'n_words': 1889}