<commit_before>#!/bin/sh

# ----------------------------
ftp_dir="/path/to/folder/on/ftp"
ftp_host="192.168.1.1"
ftp_port="21"
ftp_login="ftp_username"
ftp_pass="<PASSWORD>"
ftp_mem_file="/tmp/hd1/test/ftp_upload.mem"
ftp_log_file="/tmp/hd1/test/ftp_upload.log"
pid_file="/tmp/hd1/test/ftp_upload.pid"
# ----------------------------

record_dir="/tmp/hd1/record/"

max_d01=31
max_d02=28
max_d03=31
max_d04=30
max_d05=31
max_d06=30
max_d07=31
max_d08=31
max_d09=30
max_d10=31
max_d11=30
max_d12=31

is_server_live()
{
    ping -c1 -W2 $1 > /dev/null
    return $?
}

is_pid_exist()
{
    count=$(ps | grep $1 | wc -l)
    if [ $count -gt 1 ]; then
       return 0
    fi
    return 1
}

pid_store()
{
    echo $1 > "$pid_file"
}

pid_get()
{
    cat $pid_file
}

pid_clear()
{
    cat /dev/null > $pid_file
}

log()
{
   echo $2 $1
   echo $2 "$(date +'%Y%m%d.%H%M%S') $1" >> $ftp_log_file
}

mem_store()
{
   last_folder=$1
   last_file=$2
   echo "${last_folder}/${last_file}" > $ftp_mem_file
}

mem_get()
{
   last_folder=$(cat ${ftp_mem_file} | cut -d'/' -f1)
   last_file=$(cat ${ftp_mem_file} | cut -d'/' -f2)
   if [ -z "${last_folder}" ] || [ -z "${last_file}" ]; then
      log "[SCR] Cannot find last folder and file in $ftp_mem_file"
      log "[SRC] The file should content as: 2016Y08M01D13H|23M00S.mp4"
      exit 1
   fi
}

ftp_get_pasv_port()
{
   ret=`(sleep 1
         echo "USER ${ftp_login}"
         sleep 1
         echo "PASS ${ftp_pass}"
         sleep 1
         echo "CWD ${ftp_dir}"
         sleep 1
         echo "PASV"
         sleep 1
         echo "LIST"
         sleep 1
         echo "QUIT"
         sleep 1) | telnet ${ftp_host} ${ftp_port}`
   ret=$(echo $ret | sed -n 's/.*(\(.*\)).*/\1/p')
   #echo $ret
   a=$(echo $ret | cut -d',' -f5)
   b=$(echo $ret | cut -d',' -f6)
   echo $((a * 256 + b))
}

ftp_get_return()
{
   port=$1
   tmp_f=$2
   telnet ${ftp_host} ${port} > $tmp_f 2>&1
}

ftp_mkd()
{
   (sleep 1;
    echo "USER ${ftp_login}";
    sleep 1;
    echo "PASS ${ftp_pass}";
    sleep 1;
    echo "MKD ${ftp_dir}/$1";
    sleep 1;
    echo "QUIT";
    sleep 1 ) | telnet ${ftp_host} ${ftp_port} >> $ftp_log_file
}

ftp_upload()
{
   from_f=$1
   to_f=$2
   ftpput -u ${ftp_login} -p ${ftp_pass} -P ${ftp_port} ${ftp_host} \
          ${ftp_dir}/${to_f} ${from_f} >> $ftp_log_file 2>&1
   return $?
}

is_leap_year()
{
   year=$1
   if [ $((year % 400)) -eq 0 ]; then
      return 0
   elif [ $((year % 4)) -eq 0 ] && [ $((year % 100)) -ne 0 ]; then
      return 0
   else
      return 1
   fi
}

main()
{
   last_folder = ""
   last_file = ""

   # Here we goooooo!
   is_server_live $ftp_host
   if [ $? -ne 0 ]; then<commit_msg>Fix issue last_file not found<commit_after>#!/bin/sh

# ----------------------------
ftp_dir="/path/to/folder/on/ftp"
ftp_host="192.168.1.1"
ftp_port="21"
ftp_login="ftp_username"
ftp_pass="<PASSWORD>"
ftp_mem_file="/tmp/hd1/test/ftp_upload.mem"
ftp_log_file="/tmp/hd1/test/ftp_upload.log"
pid_file="/tmp/hd1/test/ftp_upload.pid"
# ----------------------------

record_dir="/tmp/hd1/record/"

max_d01=31
max_d02=28
max_d03=31
max_d04=30
max_d05=31
max_d06=30
max_d07=31
max_d08=31
max_d09=30
max_d10=31
max_d11=30
max_d12=31

is_server_live()
{
    ping -c1 -W2 $1 > /dev/null
    return $?
}

is_pid_exist()
{
    count=$(ps | grep $1 | wc -l)
    if [ $count -gt 1 ]; then
       return 0
    fi
    return 1
}

pid_store()
{
    echo $1 > "$pid_file"
}

pid_get()
{
    cat $pid_file
}

pid_clear()
{
    cat /dev/null > $pid_file
}

log()
{
   echo $2 $1
   echo $2 "$(date +'%Y%m%d.%H%M%S') $1" >> $ftp_log_file
}

mem_store()
{
   last_folder=$1
   last_file=$2
   echo "${last_folder}/${last_file}" > $ftp_mem_file
}

mem_get()
{
   last_folder=$(cat ${ftp_mem_file} | cut -d'/' -f1)
   last_file=$(cat ${ftp_mem_file} | cut -d'/' -f2)
   if [ -z "${last_folder}" ] || [ -z "${last_file}" ]; then
      log "[SCR] Cannot find last folder and file in $ftp_mem_file"
      log "[SRC] The file should content as: 2016Y08M01D13H|23M00S.mp4"
      exit 1
   fi
}

ftp_get_pasv_port()
{
   ret=`(sleep 1
         echo "USER ${ftp_login}"
         sleep 1
         echo "PASS ${ftp_pass}"
         sleep 1
         echo "CWD ${ftp_dir}"
         sleep 1
         echo "PASV"
         sleep 1
         echo "LIST"
         sleep 1
         echo "QUIT"
         sleep 1) | telnet ${ftp_host} ${ftp_port}`
   ret=$(echo $ret | sed -n 's/.*(\(.*\)).*/\1/p')
   #echo $ret
   a=$(echo $ret | cut -d',' -f5)
   b=$(echo $ret | cut -d',' -f6)
   echo $((a * 256 + b))
}

ftp_get_return()
{
   port=$1
   tmp_f=$2
   telnet ${ftp_host} ${port} > $tmp_f 2>&1
}

ftp_mkd()
{
   (sleep 1;
    echo "USER ${ftp_login}";
    sleep 1;
    echo "PASS ${ftp_pass}";
    sleep 1;
    echo "MKD ${ftp_dir}/$1";
    sleep 1;
    echo "QUIT";
    sleep 1 ) | telnet ${ftp_host} ${ftp_port} >> $ftp_log_file
}

ftp_upload()
{
   from_f=$1
   to_f=$2
   ftpput -u ${ftp_login} -p ${ftp_pass} -P ${ftp_port} ${ftp_host} \
          ${ftp_dir}/${to_f} ${from_f} >> $ftp_log_file 2>&1
   return $?
}

is_leap_year()
{
   year=$1
   if [ $((year % 400)) -eq 0 ]; then
      return 0
   elif [ $((year % 4)) -eq 0 ] && [ $((year % 100)) -ne 0 ]; then
      return 0
   else
      return 1
   fi
}

main()
{
   last_folder=""
   last_file=""

   # Here we goooooo!
   is_server_live $ftp_host
   if [ $? -ne 0 ]; then
{'fraction_non_alphanumeric': 0.20126723816623182, 'fraction_numerical': 0.0506895266492732, 'mean_word_length': 2.3356121814791795, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6031022', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2302, 'n_words': 509}