<commit_before>#!/bin/sh
# Copyright Abandoned 1996 TCX DataKonsult AB & Monty Program KB & Detron HB
# This file is public domain and comes with NO WARRANTY of any kind
#
# Script to start the MySQL daemon and restart it if it dies unexpectedly
#
# This should be executed in the MySQL base directory if you are using a
# binary installation that is not installed in its compile-time default
# location
#
# mysql.server works by first doing a cd to the base directory and from there
# executing mysqld_safe

# Initialize script globals
KILL_MYSQLD=1;
MYSQLD=
niceness=0
nowatch=0
mysqld_ld_preload=
mysqld_ld_library_path=
flush_caches=0
numa_interleave=0
wsrep_on=0
dry_run=0

# Initial logging status: error log is not open, and not using syslog
logging=init
want_syslog=0
syslog_tag=
user='@MYSQLD_USER@'
pid_file=
err_log=
err_log_base=
skip_err_log=0

syslog_tag_mysqld=mysqld
syslog_tag_mysqld_safe=mysqld_safe

trap '' 1 2 3 15			# we shouldn't let anyone kill us

# MySQL-specific environment variable. First off, it's not really a umask,
# it's the desired mode. Second, it follows umask(2), not umask(3) in that
# octal needs to be explicit. Our shell might be a proper sh without printf,
# multiple-base arithmetic, and binary arithmetic, so this will get ugly.
# We reject decimal values to keep things at least half-sane.
umask 007                               # fallback
UMASK="${UMASK-0640}"
fmode=`echo "$UMASK" | sed -e 's/[^0246]//g'`
octalp=`echo "$fmode"|cut -c1`
fmlen=`echo "$fmode"|wc -c|sed -e 's/ //g'`
if [ "x$octalp" != "x0" -o "x$UMASK" != "x$fmode" -o "x$fmlen" != "x5" ]
then
  fmode=0640
  echo "UMASK must be a 3-digit mode with an additional leading 0 to indicate octal." >&2
  echo "The first digit will be corrected to 6, the others may be 0, 2, 4, or 6." >&2
fi
fmode=`echo "$fmode"|cut -c3-4`
fmode="6$fmode"
if [ "x$UMASK" != "x0$fmode" ]
then
  echo "UMASK corrected from $UMASK to 0$fmode ..."
fi

defaults=
case "$1" in
    --no-defaults|--defaults-file=*|--defaults-extra-file=*)
      defaults="$1"; shift
      ;;
esac

usage () {
        cat <<EOF
Usage: $0 [OPTIONS]
  --no-defaults              Don't read the system defaults file
  --core-file-size=LIMIT     Limit core files to the specified size
  --defaults-file=FILE       Use the specified defaults file
  --defaults-extra-file=FILE Also use defaults from the specified file
  --ledir=DIRECTORY          Look for mysqld in the specified directory
  --open-files-limit=LIMIT   Limit the number of open files
  --crash-script=FILE        Script to call when mysqld crashes
  --timezone=TZ              Set the system timezone
  --malloc-lib=LIB           Preload shared library LIB if available
  --mysqld=FILE              Use the specified file as mysqld
  --mysqld-version=VERSION   Use "mysqld-VERSION" as mysqld
  --dry-run                  Simulate the start to detect errors but don't start
  --nice=NICE                Set the scheduling priority of mysqld
  --no-auto-restart          Exit after starting mysqld
  --nowatch                  Exit after starting mysqld
  --plugin-dir=DIR           Plugins are under DIR or DIR/VERSION, if
                             VERSION is given
  --skip-kill-mysqld         Don't try to kill stray mysqld processes
  --syslog                   Log messages to syslog with 'logger'
  --skip-syslog              Log messages to error log (default)
  --syslog-tag=TAG           Pass -t "mysqld-TAG" to 'logger'
  --flush-caches             Flush and purge buffers/caches before
                             starting the server
  --numa-interleave          Run mysqld with its memory interleaved
                             on all NUMA nodes

All other options are passed to the mysqld program.

EOF
        exit 1
}

my_which ()
{
  save_ifs="${IFS-UNSET}"
  IFS=:
  ret=0
  for file
  do
    for dir in $PATH
    do
      if [ -f "$dir/$file" ]
      then
        echo "$dir/$file"
        continue 2
      fi
    done

	ret=1  #signal an error
	break
  done

  if [ "$save_ifs" = UNSET ]
  then
    unset IFS
  else
    IFS="$save_ifs"
  fi
  return $ret  # Success
}

log_generic () {
  [ $dry_run -eq 1 ] && return
  priority="$1"
  shift

  msg="`date +'%y%m%d %H:%M:%S'` mysqld_safe $*"
  echo "$msg"
  case $logging in
    init) ;;  # Just echo the message, don't save it anywhere
    file) echo "$msg" >> "$err_log" ;;
    syslog) logger -t "$syslog_tag_mysqld_safe" -p "$priority" "$*" ;;
    *)
      echo "Internal program error (non-fatal):" \
           " unknown logging method '$logging'" >&2
      ;;
  esac
}

log_error () {
  log_generic daemon.error "$@" >&2
}

log_notice () {
  log_generic daemon.notice "$@"
}

eval_log_error () {
  local cmd="$1"
  case $logging in
    file) cmd="$cmd >> "`shell_quote_string "$err_log"`" 2>&1" ;;
    syslog)
      # mysqld often prefixes its messages with a timestamp, which is
      # redundant when logging to syslog (which adds its own timestamp)
      # However, we don't strip the timestamp with sed here, because
      # sed buffers output (only GNU sed supports a -u (unbuffered) option)
      # which means that messages may not get sent to syslog until the
      # mysqld process quits.
      cmd="$cmd 2>&1 | logger -t '$syslog_tag_mysqld' -p daemon.error"
      ;;
    *)
      echo "Internal program error (non-fatal):" \
           " unknown logging method '$logging'" >&2
      ;;
  esac

  if test $nowatch -eq 1
  then
    # We'd prefer to exec $cmd here, but SELinux needs to be fixed first
    #/usr/bin/logger "Running mysqld: $cmd"
    eval "$cmd &"
    exit 0
  else
    #echo "Running mysqld: [$cmd]"
    eval "$cmd"
  fi
}

shell_quote_string() {
  # This sed command makes sure that any special chars are quoted,
  # so the arg gets passed exactly to the server.
  echo "$1" | sed -e 's,\([^a-zA-Z0-9/_.=-]\),\\\1,g'
}

wsrep_pick_url() {
  [ $# -eq 0 ] && return 0

  log_error "WSREP: 'wsrep_urls' is DEPRECATED! Use wsrep_cluster_address to specify multiple addresses instead."

  if ! which nc >/dev/null; then
    log_error "ERROR: nc tool not found in PATH! Make sure you have it installed."
    return 1
  fi

  local url
  # Assuming URL in the form scheme://host:port
  # If host and port are not NULL, the liveness of URL is assumed to be tested
  # If port part is absent, the url is returned literally and unconditionally
  # If every URL has port but none is reachable, nothing is returned
  for url in `echo $@ | sed s/,/\ /g` 0; do
    local host=`echo $url | cut -d \: -f 2 | sed s/^\\\/\\\///`
    local port=`echo $url | cut -d \: -f 3`
    [ -z "$port" ] && break
    nc -z "$host" $port >/dev/null && break
  done

  if [ "$url" == "0" ]; then
    log_error "ERROR: none of the URLs in '$@' is reachable."
    return 1
  fi

  echo $url
}

# Run mysqld with --wsrep-recover and parse recovered position from log.
# Position will be stored in wsrep_start_position_opt global.
wsrep_start_position_opt=""
wsrep_recover_position() {
  local mysqld_cmd="$@"
  local euid=$(id -u)
  local ret=0

  local wr_logfile=$(mktemp $DATADIR/wsrep_recovery.XXXXXX)

  # safety checks
  if [ -z $wr_logfile ]; then
    log_error "WSREP: mktemp failed"
    return 1
  fi

  if [ -f $wr_logfile ]; then
    [ "$euid" = "0" ] && chown $user $wr_logfile
    chmod 600 $wr_logfile
  else
    log_error "WSREP: mktemp failed"
    return 1
  fi

  local wr_pidfile="$DATADIR/"`@HOSTNAME@`"-recover.pid"

  local wr_options="--log_error='$wr_logfile' --pid-file='$wr_pidfile'"

  log_notice "WSREP: Running position recovery with $wr_options"

  eval_log_error "$mysqld_cmd --wsrep_recover $wr_options"

  local rp="$(grep 'WSREP: Recovered position:' $wr_logfile)"
  if [ -z "$rp" ]; then
    local skipped="$(grep WSREP $wr_logfile | grep 'skipping position recovery')"
    if [ -z "$skipped" ]; then
      log_error "WSREP: Failed to recover position: '`cat $wr_logfile`'"
      ret=1
    else
      log_notice "WSREP: Position recovery skipped"
    fi
  else
    local start_pos="$(echo $rp | sed 's/.*WSREP\:\ Recovered\ position://' \
        | sed 's/^[ \t]*//')"
    log_notice "WSREP: Recovered position $start_pos"
    wsrep_start_position_opt="--wsrep_start_position=$start_pos"
  fi

  [ $ret -eq 0 ] && rm $wr_logfile

  return $ret
}

parse_arguments() {
  for arg do
    val=`echo "$arg" | sed -e "s;--[^=]*=;;"`
    case "$arg" in
      --crash[-_]script=*) CRASH_SCRIPT="$val" ;;
      # these get passed explicitly to mysqld
      --basedir=*) MY_BASEDIR_VERSION="$val" ;;
      --datadir=*|--data=*) DATADIR="$val" ;;
      --pid[-_]file=*) pid_file="$val" ;;
      --plugin[-_]dir=*) PLUGIN_DIR="$val" ;;
      --user=*) user="$val"; SET_USER=1 ;;
      --log[-_]basename=*|--hostname=*|--loose[-_]log[-_]basename=*)
        pid_file="$val.pid";
	err_log_base="$val";
	;;

      # these might have been set in a [mysqld_safe] section of my.cnf
      # they are added to mysqld command line to override settings from my.cnf
      --skip[-_]log[-_]error)
        err_log=;
        skip_err_log=1;
        ;;
      --log[-_]error=*)
        err_log="$val";
        skip_err_log=0;
        ;;
      --port=*) mysql_tcp_port="$val" ;;
      --socket=*) mysql_unix_port="$val" ;;

      # mysqld_safe-specific options - must be set in my.cnf ([mysqld_safe])!
      --core[-_]file[-_]size=*) core_file_size="$val" ;;
      --ledir=*) ledir="$val" ;;
      --malloc[-_]lib=*) set_malloc_lib "$val" ;;
      --mysqld=*) MYSQLD="$val" ;;
      --mysqld[-_]version=*)
        if test -n "$val"
        then
          MYSQLD="mysqld-$val"
          PLUGIN_VARIANT="/$val"
        else
          MYSQLD="mysqld"
        fi
        ;;
      --dry[-_]run) dry_run=1 ;;
      --nice=*) niceness="$val" ;;
      --nowatch|--no[-_]watch|--no[-_]auto[-_]restart) nowatch=1 ;;
      --open[-_]files[-_]limit=*) open_files="$val" ;;
      --skip[-_]kill[-_]mysqld*) KILL_MYSQLD=0 ;;
      --syslog) want_syslog=1 ;;
      --skip[-_]syslog) want_syslog=0 ;;
      --syslog[-_]tag=*) syslog_tag="$val" ;;
      --timezone=*) TZ="$val"; export TZ; ;;
      --flush[-_]caches) flush_caches=1 ;;
      --numa[-_]interleave) numa_interleave=1 ;;
      --wsrep[-_]on) wsrep_on=1 ;;
      --skip[-_]wsrep[-_]on) wsrep_on=0 ;;
      --wsrep[-_]on=*)
        if echo $val | grep -iq '\(ON\|1\)'; then
          wsrep_on=1
        else
          wsrep_on=0
        fi
        ;;
      --wsrep[-_]urls=*) wsrep_urls="$val"; ;;
      --wsrep[-_]provider=*)
        if test -n "$val" && test "$val" != "none"
        then
          wsrep_restart=1
        fi
        append_arg_to_args "$arg"
        ;;

      --help) usage ;;

      *)<commit_msg>MDEV-10230: --wsrep_on option no longer passed through by mysqld_safe<commit_after>#!/bin/sh
# Copyright Abandoned 1996 TCX DataKonsult AB & Monty Program KB & Detron HB
# This file is public domain and comes with NO WARRANTY of any kind
#
# Script to start the MySQL daemon and restart it if it dies unexpectedly
#
# This should be executed in the MySQL base directory if you are using a
# binary installation that is not installed in its compile-time default
# location
#
# mysql.server works by first doing a cd to the base directory and from there
# executing mysqld_safe

# Initialize script globals
KILL_MYSQLD=1;
MYSQLD=
niceness=0
nowatch=0
mysqld_ld_preload=
mysqld_ld_library_path=
flush_caches=0
numa_interleave=0
wsrep_on=0
dry_run=0

# Initial logging status: error log is not open, and not using syslog
logging=init
want_syslog=0
syslog_tag=
user='@MYSQLD_USER@'
pid_file=
err_log=
err_log_base=
skip_err_log=0

syslog_tag_mysqld=mysqld
syslog_tag_mysqld_safe=mysqld_safe

trap '' 1 2 3 15			# we shouldn't let anyone kill us

# MySQL-specific environment variable. First off, it's not really a umask,
# it's the desired mode. Second, it follows umask(2), not umask(3) in that
# octal needs to be explicit. Our shell might be a proper sh without printf,
# multiple-base arithmetic, and binary arithmetic, so this will get ugly.
# We reject decimal values to keep things at least half-sane.
umask 007                               # fallback
UMASK="${UMASK-0640}"
fmode=`echo "$UMASK" | sed -e 's/[^0246]//g'`
octalp=`echo "$fmode"|cut -c1`
fmlen=`echo "$fmode"|wc -c|sed -e 's/ //g'`
if [ "x$octalp" != "x0" -o "x$UMASK" != "x$fmode" -o "x$fmlen" != "x5" ]
then
  fmode=0640
  echo "UMASK must be a 3-digit mode with an additional leading 0 to indicate octal." >&2
  echo "The first digit will be corrected to 6, the others may be 0, 2, 4, or 6." >&2
fi
fmode=`echo "$fmode"|cut -c3-4`
fmode="6$fmode"
if [ "x$UMASK" != "x0$fmode" ]
then
  echo "UMASK corrected from $UMASK to 0$fmode ..."
fi

defaults=
case "$1" in
    --no-defaults|--defaults-file=*|--defaults-extra-file=*)
      defaults="$1"; shift
      ;;
esac

usage () {
        cat <<EOF
Usage: $0 [OPTIONS]
  --no-defaults              Don't read the system defaults file
  --core-file-size=LIMIT     Limit core files to the specified size
  --defaults-file=FILE       Use the specified defaults file
  --defaults-extra-file=FILE Also use defaults from the specified file
  --ledir=DIRECTORY          Look for mysqld in the specified directory
  --open-files-limit=LIMIT   Limit the number of open files
  --crash-script=FILE        Script to call when mysqld crashes
  --timezone=TZ              Set the system timezone
  --malloc-lib=LIB           Preload shared library LIB if available
  --mysqld=FILE              Use the specified file as mysqld
  --mysqld-version=VERSION   Use "mysqld-VERSION" as mysqld
  --dry-run                  Simulate the start to detect errors but don't start
  --nice=NICE                Set the scheduling priority of mysqld
  --no-auto-restart          Exit after starting mysqld
  --nowatch                  Exit after starting mysqld
  --plugin-dir=DIR           Plugins are under DIR or DIR/VERSION, if
                             VERSION is given
  --skip-kill-mysqld         Don't try to kill stray mysqld processes
  --syslog                   Log messages to syslog with 'logger'
  --skip-syslog              Log messages to error log (default)
  --syslog-tag=TAG           Pass -t "mysqld-TAG" to 'logger'
  --flush-caches             Flush and purge buffers/caches before
                             starting the server
  --numa-interleave          Run mysqld with its memory interleaved
                             on all NUMA nodes

All other options are passed to the mysqld program.

EOF
        exit 1
}

my_which ()
{
  save_ifs="${IFS-UNSET}"
  IFS=:
  ret=0
  for file
  do
    for dir in $PATH
    do
      if [ -f "$dir/$file" ]
      then
        echo "$dir/$file"
        continue 2
      fi
    done

	ret=1  #signal an error
	break
  done

  if [ "$save_ifs" = UNSET ]
  then
    unset IFS
  else
    IFS="$save_ifs"
  fi
  return $ret  # Success
}

log_generic () {
  [ $dry_run -eq 1 ] && return
  priority="$1"
  shift

  msg="`date +'%y%m%d %H:%M:%S'` mysqld_safe $*"
  echo "$msg"
  case $logging in
    init) ;;  # Just echo the message, don't save it anywhere
    file) echo "$msg" >> "$err_log" ;;
    syslog) logger -t "$syslog_tag_mysqld_safe" -p "$priority" "$*" ;;
    *)
      echo "Internal program error (non-fatal):" \
           " unknown logging method '$logging'" >&2
      ;;
  esac
}

log_error () {
  log_generic daemon.error "$@" >&2
}

log_notice () {
  log_generic daemon.notice "$@"
}

eval_log_error () {
  local cmd="$1"
  case $logging in
    file) cmd="$cmd >> "`shell_quote_string "$err_log"`" 2>&1" ;;
    syslog)
      # mysqld often prefixes its messages with a timestamp, which is
      # redundant when logging to syslog (which adds its own timestamp)
      # However, we don't strip the timestamp with sed here, because
      # sed buffers output (only GNU sed supports a -u (unbuffered) option)
      # which means that messages may not get sent to syslog until the
      # mysqld process quits.
      cmd="$cmd 2>&1 | logger -t '$syslog_tag_mysqld' -p daemon.error"
      ;;
    *)
      echo "Internal program error (non-fatal):" \
           " unknown logging method '$logging'" >&2
      ;;
  esac

  if test $nowatch -eq 1
  then
    # We'd prefer to exec $cmd here, but SELinux needs to be fixed first
    #/usr/bin/logger "Running mysqld: $cmd"
    eval "$cmd &"
    exit 0
  else
    #echo "Running mysqld: [$cmd]"
    eval "$cmd"
  fi
}

shell_quote_string() {
  # This sed command makes sure that any special chars are quoted,
  # so the arg gets passed exactly to the server.
  echo "$1" | sed -e 's,\([^a-zA-Z0-9/_.=-]\),\\\1,g'
}

wsrep_pick_url() {
  [ $# -eq 0 ] && return 0

  log_error "WSREP: 'wsrep_urls' is DEPRECATED! Use wsrep_cluster_address to specify multiple addresses instead."

  if ! which nc >/dev/null; then
    log_error "ERROR: nc tool not found in PATH! Make sure you have it installed."
    return 1
  fi

  local url
  # Assuming URL in the form scheme://host:port
  # If host and port are not NULL, the liveness of URL is assumed to be tested
  # If port part is absent, the url is returned literally and unconditionally
  # If every URL has port but none is reachable, nothing is returned
  for url in `echo $@ | sed s/,/\ /g` 0; do
    local host=`echo $url | cut -d \: -f 2 | sed s/^\\\/\\\///`
    local port=`echo $url | cut -d \: -f 3`
    [ -z "$port" ] && break
    nc -z "$host" $port >/dev/null && break
  done

  if [ "$url" == "0" ]; then
    log_error "ERROR: none of the URLs in '$@' is reachable."
    return 1
  fi

  echo $url
}

# Run mysqld with --wsrep-recover and parse recovered position from log.
# Position will be stored in wsrep_start_position_opt global.
wsrep_start_position_opt=""
wsrep_recover_position() {
  local mysqld_cmd="$@"
  local euid=$(id -u)
  local ret=0

  local wr_logfile=$(mktemp $DATADIR/wsrep_recovery.XXXXXX)

  # safety checks
  if [ -z $wr_logfile ]; then
    log_error "WSREP: mktemp failed"
    return 1
  fi

  if [ -f $wr_logfile ]; then
    [ "$euid" = "0" ] && chown $user $wr_logfile
    chmod 600 $wr_logfile
  else
    log_error "WSREP: mktemp failed"
    return 1
  fi

  local wr_pidfile="$DATADIR/"`@HOSTNAME@`"-recover.pid"

  local wr_options="--log_error='$wr_logfile' --pid-file='$wr_pidfile'"

  log_notice "WSREP: Running position recovery with $wr_options"

  eval_log_error "$mysqld_cmd --wsrep_recover $wr_options"

  local rp="$(grep 'WSREP: Recovered position:' $wr_logfile)"
  if [ -z "$rp" ]; then
    local skipped="$(grep WSREP $wr_logfile | grep 'skipping position recovery')"
    if [ -z "$skipped" ]; then
      log_error "WSREP: Failed to recover position: '`cat $wr_logfile`'"
      ret=1
    else
      log_notice "WSREP: Position recovery skipped"
    fi
  else
    local start_pos="$(echo $rp | sed 's/.*WSREP\:\ Recovered\ position://' \
        | sed 's/^[ \t]*//')"
    log_notice "WSREP: Recovered position $start_pos"
    wsrep_start_position_opt="--wsrep_start_position=$start_pos"
  fi

  [ $ret -eq 0 ] && rm $wr_logfile

  return $ret
}

parse_arguments() {
  for arg do
    val=`echo "$arg" | sed -e "s;--[^=]*=;;"`
    case "$arg" in
      --crash[-_]script=*) CRASH_SCRIPT="$val" ;;
      # these get passed explicitly to mysqld
      --basedir=*) MY_BASEDIR_VERSION="$val" ;;
      --datadir=*|--data=*) DATADIR="$val" ;;
      --pid[-_]file=*) pid_file="$val" ;;
      --plugin[-_]dir=*) PLUGIN_DIR="$val" ;;
      --user=*) user="$val"; SET_USER=1 ;;
      --log[-_]basename=*|--hostname=*|--loose[-_]log[-_]basename=*)
        pid_file="$val.pid";
	err_log_base="$val";
	;;

      # these might have been set in a [mysqld_safe] section of my.cnf
      # they are added to mysqld command line to override settings from my.cnf
      --skip[-_]log[-_]error)
        err_log=;
        skip_err_log=1;
        ;;
      --log[-_]error=*)
        err_log="$val";
        skip_err_log=0;
        ;;
      --port=*) mysql_tcp_port="$val" ;;
      --socket=*) mysql_unix_port="$val" ;;

      # mysqld_safe-specific options - must be set in my.cnf ([mysqld_safe])!
      --core[-_]file[-_]size=*) core_file_size="$val" ;;
      --ledir=*) ledir="$val" ;;
      --malloc[-_]lib=*) set_malloc_lib "$val" ;;
      --mysqld=*) MYSQLD="$val" ;;
      --mysqld[-_]version=*)
        if test -n "$val"
        then
          MYSQLD="mysqld-$val"
          PLUGIN_VARIANT="/$val"
        else
          MYSQLD="mysqld"
        fi
        ;;
      --dry[-_]run) dry_run=1 ;;
      --nice=*) niceness="$val" ;;
      --nowatch|--no[-_]watch|--no[-_]auto[-_]restart) nowatch=1 ;;
      --open[-_]files[-_]limit=*) open_files="$val" ;;
      --skip[-_]kill[-_]mysqld*) KILL_MYSQLD=0 ;;
      --syslog) want_syslog=1 ;;
      --skip[-_]syslog) want_syslog=0 ;;
      --syslog[-_]tag=*) syslog_tag="$val" ;;
      --timezone=*) TZ="$val"; export TZ; ;;
      --flush[-_]caches) flush_caches=1 ;;
      --numa[-_]interleave) numa_interleave=1 ;;
      --wsrep[-_]on)
        wsrep_on=1
        append_arg_to_args "$arg"
        ;;
      --skip[-_]wsrep[-_]on)
        wsrep_on=0
        append_arg_to_args "$arg"
        ;;
      --wsrep[-_]on=*)
        if echo $val | grep -iq '\(ON\|1\)'; then
          wsrep_on=1
        else
          wsrep_on=0
        fi
        append_arg_to_args "$arg"
        ;;
      --wsrep[-_]urls=*) wsrep_urls="$val"; ;;
      --wsrep[-_]provider=*)
        if test -n "$val" && test "$val" != "none"
        then
          wsrep_restart=1
        fi
        append_arg_to_args "$arg"
        ;;

      --help) usage ;;

      *)
{'fraction_non_alphanumeric': 0.13668897564603086, 'fraction_numerical': 0.010271425915597696, 'mean_word_length': 2.5135532331809274, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1513439', 'n_tokens_mistral': 7843, 'n_tokens_neox': 7392, 'n_words': 2258}