<commit_before>#!/bin/sh

######################################################################
#
# STWSRCH.SH : Search Twitters Which Match With Given Keywords
#              (on Streaming API Mode)
#
# Written by <NAME> (@shellshoccarjpn) on 2017-02-10
#
# This is a public-domain software (CC0). It measns that all of the
# people can use this for any purposes with no restrictions at all.
# By the way, I am fed up the side effects which are broght about by
# the major licenses.
#
######################################################################


######################################################################
# Initial Configuration
######################################################################

# === Initialize shell environment ===================================
set -um # "-m" is required to use "fg" command
umask 0022
export LC_ALL=C
export PATH="$(command -p getconf PATH)${PATH:+:}${PATH:-}"

# === Define the functions for printing usage and exiting ============
print_usage_and_exit () {
  cat <<-USAGE 1>&2
	Usage   : ${0##*/} [options] <keyword> [keyword ...]
	Options :
	          -u <user_ID>[,user_ID...]|--follow=<user_ID>[,user_ID...]
	          -l <lat>,<long>[,<...>]  |--locations=<lat>,<long>[,<...>]
	          -v                       |--verbose
	          --rawout=<filepath_for_writing_JSON_data>
	          --rawonly
	          --timeout=<waiting_seconds_to_connect>
	Version : 2017-02-10 02:57:49 JST
	USAGE
  exit 1
}
exit_trap() {
  trap - EXIT HUP INT QUIT PIPE ALRM TERM
  [ -d "${Tmp:-}" ] && rm -rf "${Tmp%/*}/_${Tmp##*/_}"
  case "$webcmdpid" in '') sleep 1; set_webcmdpid;; esac
  case "$webcmdpid" in
    '-'*) :                                 ;;
       *) echo 'Flush buffered data...' 1>&3
          kill $webcmdpid 2>/dev/null && fg
          webcmdpid=-1
          exec 1>&3 2>&4 3>&- 4>&-          ;;
  esac
  exit ${1:-0}
}
error_exit() {
  [ -n "$2" ] && echo "${0##*/}: $2" 1>&2
  exit_trap $1
}

# === Define one more special function for exiting politely ==========
# --- a variable for the function ------------------------------------
webcmdpid=-1 # PID which the command accesing Twitter API is using
             # <0 .... No process exists now or finished already.
             #           >>> So you may exit immediately.
             # null .. Process will be created soon.
             #           >>> You must wait for being set a PID to $webcmdpid
             # >=0 ... The process accessing Twitter API now is $webcmdpid
             #           >>> You must kill it before exiting
# --- FUNC : Investigate and set PID of cURL/Wget command called by itself
set_webcmdpid() {
  webcmdpid=`case $(uname) in                                             #
               CYGWIN*) ps -af                                      |     #
                        awk '{c=$6;sub(/^.*\//,"",c);print $3,$2,c}';;    #
                     *) ps -Ao ppid,pid,comm                        ;;    #
             esac                                                         |
             grep -v '^[^0-9]*PPID'                                       |
             sort -k 1n,1 -k 2n,2                                         |
             awk 'BEGIN    {ppid0="" ;         }                          #
                  ppid0!=$1{print "-";         }                          #
                  {         print    ;ppid0=$1;}'                         |<commit_msg>BIN/stwsrch.sh: rewrite a part of comment<commit_after>#!/bin/sh

######################################################################
#
# STWSRCH.SH : Search Twitters Which Match With Given Keywords
#              (on Streaming API Mode)
#
# Written by <NAME> (@shellshoccarjpn) on 2017-02-11
#
# This is a public-domain software (CC0). It measns that all of the
# people can use this for any purposes with no restrictions at all.
# By the way, I am fed up the side effects which are broght about by
# the major licenses.
#
######################################################################


######################################################################
# Initial Configuration
######################################################################

# === Initialize shell environment ===================================
set -um # "-m" is required to use "fg" command
umask 0022
export LC_ALL=C
export PATH="$(command -p getconf PATH)${PATH:+:}${PATH:-}"

# === Define the functions for printing usage and exiting ============
print_usage_and_exit () {
  cat <<-USAGE 1>&2
	Usage   : ${0##*/} [options] <keyword> [keyword ...]
	Options :
	          -u <user_ID>[,user_ID...]|--follow=<user_ID>[,user_ID...]
	          -l <lat>,<long>[,<...>]  |--locations=<lat>,<long>[,<...>]
	          -v                       |--verbose
	          --rawout=<filepath_for_writing_JSON_data>
	          --rawonly
	          --timeout=<waiting_seconds_to_connect>
	Version : 2017-02-11 11:56:19 JST
	USAGE
  exit 1
}
exit_trap() {
  trap - EXIT HUP INT QUIT PIPE ALRM TERM
  [ -d "${Tmp:-}" ] && rm -rf "${Tmp%/*}/_${Tmp##*/_}"
  case "$webcmdpid" in '') sleep 1; set_webcmdpid;; esac
  case "$webcmdpid" in
    '-'*) :                                 ;;
       *) echo 'Flush buffered data...' 1>&3
          kill $webcmdpid 2>/dev/null && fg
          webcmdpid=-1
          exec 1>&3 2>&4 3>&- 4>&-          ;;
  esac
  exit ${1:-0}
}
error_exit() {
  [ -n "$2" ] && echo "${0##*/}: $2" 1>&2
  exit_trap $1
}

# === Define one more special function for exiting politely ==========
# --- a variable for the function ------------------------------------
webcmdpid=-1 # PID which the command accesing Twitter API is using
             # <0 .... No process exists now or finished already.
             #           >>> So you may exit immediately.
             # null .. Process will be created soon.
             #           >>> You must call set_webcmdpid and retry to refer
             #               $webcmdpid before exiting
             # >=0 ... The process accessing Twitter API now is $webcmdpid
             #           >>> You must kill it before exiting
# --- FUNC : Investigate and set PID of cURL/Wget command called by itself
set_webcmdpid() {
  webcmdpid=`case $(uname) in                                             #
               CYGWIN*) ps -af                                      |     #
                        awk '{c=$6;sub(/^.*\//,"",c);print $3,$2,c}';;    #
                     *) ps -Ao ppid,pid,comm                        ;;    #
             esac                                                         |
             grep -v '^[^0-9]*PPID'                                       |
             sort -k 1n,1 -k 2n,2                                         |
             awk 'BEGIN    {ppid0="" ;         }                          #
                  ppid0!=$1{print "-";         }                          #
                  {         print    ;ppid0=$1;}'                         |
{'fraction_non_alphanumeric': 0.23030477285796436, 'fraction_numerical': 0.01926394479585969, 'mean_word_length': 1.5891328619278005, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1248194', 'n_tokens_mistral': 2038, 'n_tokens_neox': 1853, 'n_words': 558}