<commit_before>#!/bin/sh
# shellcheck shell=dash disable=SC2169,SC3010,SC3036 source=/dev/null
#
# General purpose shell script to prepare and install a firmware update
# either archived or unarchived, verifies its validity and installs it
# unattended accordingly.
#
# Copyright (c) 2018-2021 <NAME> <<EMAIL>>
# Apache 2.0 License applies
#

######
# function that is called with the filename containing
# the firmware update either archived or unarchived.
fwprepare()
{
  filename=${1}

  echo -ne "[1/7] Checking uploaded data... "
  # check if filename exists
  if [[ ! -f "${filename}" ]]; then
    echo "ERROR ('${filename}' exists)"
    exit 1
  fi

  # check file size to be > 0
  FILESIZE=$(stat -c%s "${filename}" 2>/dev/null)
  if [[ -z "${FILESIZE}" ]] || [[ "${FILESIZE}" -le 0 ]]; then
    echo "ERROR (filesize: ${FILESIZE})"
    exit 1
  fi
  echo "${FILESIZE} bytes received, OK<br/>"

  ########
  # calculate/output sha256 checksum of uploaded data
  echo -ne "[2/7] Calculating SHA256 checksum..."

  # start a progress bar outputing dots every few seconds
  awk 'BEGIN{while(1){printf".";fflush();system("sleep 3");}}' &
  PROGRESS_PID=$!
  # shellcheck disable=SC2064
  trap "kill ${PROGRESS_PID}; rm -f /tmp/.runningFirmwareUpdate" EXIT

  if ! CHKSUM=$(/usr/bin/sha256sum "${filename}" 2>/dev/null) || [[ -z "${CHKSUM}" ]]; then
    echo "ERROR (sha256sum)"
    exit 1
  fi
  # stop the progress output
  kill ${PROGRESS_PID} && trap "rm -f /tmp/.runningFirmwareUpdate" EXIT

  echo "$(echo "${CHKSUM}" | awk '{ print $1 }')<br/>"

  ########
  # create tmpdir and output available disk space
  echo -ne "[3/7] Checking free disk space... "
  TMPDIR="${filename}-dir"
  if ! mkdir -p "${TMPDIR}"; then
    echo "ERROR: (mkdir tmpdir)"
    exit 1
  fi

  # output
  AVAILSPACE=$(/bin/df -B1 "${TMPDIR}" | tail -1 | awk '{ print $4 }')
  echo -ne "${AVAILSPACE} bytes, "

  # check if > 0 bytes available
  if [[ -z "${AVAILSPACE}" ]] || [[ "${AVAILSPACE}" -le 0 ]]; then
    echo "ERROR: not enough disk space available!"
    exit 1
  fi
  echo "OK<br/>"

  ########
  # create tmpdir and output available disk space
  echo -ne "[4/7] Checking storage performance..."

  # start a progress bar outputing dots every few seconds
  awk 'BEGIN{while(1){printf".";fflush();system("sleep 3");}}' &
  PROGRESS_PID=$!
  # shellcheck disable=SC2064
  trap "kill ${PROGRESS_PID}; rm -f /tmp/.runningFirmwareUpdate" EXIT

  # run the file i/o test using 'fio' which emulates a
  # Apps Class A1 performance test
  if ! RES=$(/usr/bin/fio --output-format=terse --max-jobs=4 /usr/share/agnostics/sd_bench.fio | cut -f 3,7,8,48,49 -d";" -) || [[ -z "${RES}" ]]; then
    echo "WARNING: performance test failed"
    # stop the progress output
    kill ${PROGRESS_PID} && trap "rm -f /tmp/.runningFirmwareUpdate" EXIT
  else
    # stop the progress output
    kill ${PROGRESS_PID} && trap "rm -f /tmp/.runningFirmwareUpdate" EXIT
    rm -f /usr/local/tmp/sd.test.file

    swri=$(echo "${RES}" | head -n 2 | tail -n 1 | cut -d ";" -f 4)
    swrimb=$(echo ${swri} | awk '{printf "%.2f",($1/1000)}')
    rwri=$(echo "$RES" | head -n 3 | tail -n 1 | cut -d ";" -f 5)
    rrea=$(echo "$RES" | head -n 4 | tail -n 1 | cut -d ";" -f 3)
    pass=0

    # sequential write check
    echo -ne "sequential write: ${swrimb} MB/s"
    if [[ "${swri}" -lt 10000 ]]; then
      echo -ne " (WARN! < 10.0 MB/s), "
      pass=1
    else
      echo -ne " (OK: > 10.0 MB/s), "
    fi

    # random write check
    echo -ne "random write: ${rwri} IOPS"
    if [[ "${rwri}" -lt 500 ]]; then
      echo -ne " (WARN! < 500 IOPS), "
      pass=1
    else
      echo -ne " (OK: > 500 IOPS), "
    fi
<commit_msg>fixed shellcheck linter error.<commit_after>#!/bin/sh
# shellcheck shell=dash disable=SC2169,SC3010,SC3036 source=/dev/null
#
# General purpose shell script to prepare and install a firmware update
# either archived or unarchived, verifies its validity and installs it
# unattended accordingly.
#
# Copyright (c) 2018-2021 <NAME> <<EMAIL>>
# Apache 2.0 License applies
#

######
# function that is called with the filename containing
# the firmware update either archived or unarchived.
fwprepare()
{
  filename=${1}

  echo -ne "[1/7] Checking uploaded data... "
  # check if filename exists
  if [[ ! -f "${filename}" ]]; then
    echo "ERROR ('${filename}' exists)"
    exit 1
  fi

  # check file size to be > 0
  FILESIZE=$(stat -c%s "${filename}" 2>/dev/null)
  if [[ -z "${FILESIZE}" ]] || [[ "${FILESIZE}" -le 0 ]]; then
    echo "ERROR (filesize: ${FILESIZE})"
    exit 1
  fi
  echo "${FILESIZE} bytes received, OK<br/>"

  ########
  # calculate/output sha256 checksum of uploaded data
  echo -ne "[2/7] Calculating SHA256 checksum..."

  # start a progress bar outputing dots every few seconds
  awk 'BEGIN{while(1){printf".";fflush();system("sleep 3");}}' &
  PROGRESS_PID=$!
  # shellcheck disable=SC2064
  trap "kill ${PROGRESS_PID}; rm -f /tmp/.runningFirmwareUpdate" EXIT

  if ! CHKSUM=$(/usr/bin/sha256sum "${filename}" 2>/dev/null) || [[ -z "${CHKSUM}" ]]; then
    echo "ERROR (sha256sum)"
    exit 1
  fi
  # stop the progress output
  kill ${PROGRESS_PID} && trap "rm -f /tmp/.runningFirmwareUpdate" EXIT

  echo "$(echo "${CHKSUM}" | awk '{ print $1 }')<br/>"

  ########
  # create tmpdir and output available disk space
  echo -ne "[3/7] Checking free disk space... "
  TMPDIR="${filename}-dir"
  if ! mkdir -p "${TMPDIR}"; then
    echo "ERROR: (mkdir tmpdir)"
    exit 1
  fi

  # output
  AVAILSPACE=$(/bin/df -B1 "${TMPDIR}" | tail -1 | awk '{ print $4 }')
  echo -ne "${AVAILSPACE} bytes, "

  # check if > 0 bytes available
  if [[ -z "${AVAILSPACE}" ]] || [[ "${AVAILSPACE}" -le 0 ]]; then
    echo "ERROR: not enough disk space available!"
    exit 1
  fi
  echo "OK<br/>"

  ########
  # create tmpdir and output available disk space
  echo -ne "[4/7] Checking storage performance..."

  # start a progress bar outputing dots every few seconds
  awk 'BEGIN{while(1){printf".";fflush();system("sleep 3");}}' &
  PROGRESS_PID=$!
  # shellcheck disable=SC2064
  trap "kill ${PROGRESS_PID}; rm -f /tmp/.runningFirmwareUpdate" EXIT

  # run the file i/o test using 'fio' which emulates a
  # Apps Class A1 performance test
  if ! RES=$(/usr/bin/fio --output-format=terse --max-jobs=4 /usr/share/agnostics/sd_bench.fio | cut -f 3,7,8,48,49 -d";" -) || [[ -z "${RES}" ]]; then
    echo "WARNING: performance test failed"
    # stop the progress output
    kill ${PROGRESS_PID} && trap "rm -f /tmp/.runningFirmwareUpdate" EXIT
  else
    # stop the progress output
    kill ${PROGRESS_PID} && trap "rm -f /tmp/.runningFirmwareUpdate" EXIT
    rm -f /usr/local/tmp/sd.test.file

    swri=$(echo "${RES}" | head -n 2 | tail -n 1 | cut -d ";" -f 4)
    swrimb=$(echo "${swri}" | awk '{printf "%.2f",($1/1000)}')
    rwri=$(echo "$RES" | head -n 3 | tail -n 1 | cut -d ";" -f 5)
    rrea=$(echo "$RES" | head -n 4 | tail -n 1 | cut -d ";" -f 3)
    pass=0

    # sequential write check
    echo -ne "sequential write: ${swrimb} MB/s"
    if [[ "${swri}" -lt 10000 ]]; then
      echo -ne " (WARN! < 10.0 MB/s), "
      pass=1
    else
      echo -ne " (OK: > 10.0 MB/s), "
    fi

    # random write check
    echo -ne "random write: ${rwri} IOPS"
    if [[ "${rwri}" -lt 500 ]]; then
      echo -ne " (WARN! < 500 IOPS), "
      pass=1
    else
      echo -ne " (OK: > 500 IOPS), "
    fi

{'fraction_non_alphanumeric': 0.1756446604563251, 'fraction_numerical': 0.031591737545565005, 'mean_word_length': 2.9935309973045823, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3406389', 'n_tokens_mistral': 2816, 'n_tokens_neox': 2585, 'n_words': 847}