<commit_before>#!/usr/bin/env bash
# dijnet.hu invoice downloader [https://github.com/wolandmaster/dijnet-dump]
# Copyright (c) 2016-2021 <NAME> and others
# This software may be distributed under the terms of the Apache 2.0 license

if ! type xmllint wget xxd &>/dev/null; then
  echo "Dependency missing! Please install them:" >&2
  echo "- debian/ubuntu: apt-get install wget xxd libxml2-utils" >&2
  echo "- fedora: dnf install wget vim-common findutils" >&2
  echo "- openwrt: opkg install bash wget xxd libxml2-utils" >&2
  exit 1
fi

die() {
  EXIT_CODE="$?" && echo -e "$@" >&2 && exit 1
}

absolute_path() {
  pushd . >/dev/null && cd "$(dirname "$1")" && pwd -P && popd >/dev/null
}

xpath() {
  xmllint --html --xpath "$1" - 2>/dev/null
}

unaccent() {
  xxd -ps | tr -d '\n' | sed 's/\(..\)/ \1/g
    s/e1/61/g;s/c1/41/g;s/e9/65/g;s/c9/45/g;s/ed/69/g;s/cd/49/g;s/f3/6f/g;s/d3/4f/g;s/f5/6f/g
    s/d5/4f/g;s/f6/6f/g;s/d6/4f/g;s/fa/75/g;s/da/55/g;s/fb/75/g;s/db/55/g;s/fc/75/g;s/dc/55/g
    s/c3 a1/61/g;s/c3 81/41/g;s/c3 a9/65/g;s/c3 89/45/g;s/c3 ad/69/g;s/c3 8d/49/g
    s/c3 b3/6f/g;s/c3 93/4f/g;s/c3 b6/6f/g;s/c3 96/4f/g;s/c5 91/6f/g;s/c5 90/4f/g
    s/c3 ba/75/g;s/c3 9a/55/g;s/c3 bc/75/g;s/c3 9c/55/g;s/c5 b1/75/g;s/c5 b0/55/g' \
  | xxd -r -ps | sed 's/&\(.\)\(acute\|uml\|dblac\);/\1/g;s/&nbsp;/ /g'
}

dijnet() {
  URL_POSTFIX="$1"; shift; local IFS=""; POST_DATA="$*"
  wget --quiet --output-document=- --post-data "${POST_DATA}" --no-check-certificate \
       --load-cookies "${COOKIES}" --save-cookies "${COOKIES}" --keep-session-cookies \
       "${DIJNET_BASE_URL}/${URL_POSTFIX}"
}

invoice_data() {
  local IFS="|"; FILTER=$(sed 's/|/" or text()="/g' <<<"$*")
  unaccent <<<"${DOWNLOAD_PAGE}" | xpath '//label[text()="'"${FILTER}"'"]/../following-sibling::td[1]//text()'
}

download_internal_links() {
  LINKS=$(xpath '//a[contains(@class, "xt_link__download")]/@href' | sed 's/href="\([^"]*\)"/\1 /g')
  for LINK in ${LINKS}; do
    grep -q "^http" <<<"${LINK}" && continue
    wget --quiet --load-cookies "${COOKIES}" --content-disposition --no-clobber --no-check-certificate \
         --directory-prefix "${FIXED_TARGET_FOLDER}" "${DIJNET_BASE_URL}/ekonto/control/${LINK}"
  done
}

progress() {
  local PROVIDER_NAME=$(sed 's/ ()$//' <<<"${INVOICE_PROVIDER} (${INVOICE_PROVIDER_ALIAS})")
  if type pv &>/dev/null; then
    pv -N "download \"${PROVIDER_NAME}\", total: ${INVOICE_COUNT}, current" \
       -W -b -p -l -t -e -s "${INVOICE_COUNT}" >/dev/null
  else
    xargs -n 1 echo -ne "\033[2K\rdownload \"${PROVIDER_NAME}\", total: ${INVOICE_COUNT}, current:"; echo
  fi
}

set -o pipefail; export LANG=C LC_ALL=C
. "$(absolute_path "$0")/dijnet-dump.conf" || die "ERROR: config file (dijnet-dump.conf) missing"
[[ "$1" == "-d" ]] && DEBUG_LOG="yes" && shift
DIJNET_USERNAME="${1:-${DIJNET_USERNAME}}"
[[ -z "${DIJNET_USERNAME}" ]] && die "usage: $(basename "$0") [-d] username"
[[ -z "${DIJNET_PASSWORD}" ]] && read -r -s -p "password: " DIJNET_PASSWORD && echo
if [[ "${DEBUG_LOG}" == "yes" ]]; then
  exec 3> >(sed 's/'"${DIJNET_USERNAME}"'/<USERNAME>/g;s/'"${DIJNET_PASSWORD}"'/********/g' >dijnet-dump.log)
  export BASH_XTRACEFD="3"
  set -x
fi<commit_msg>Use gsed when exists (required for macOS): #33<commit_after>#!/usr/bin/env bash
# dijnet.hu invoice downloader [https://github.com/wolandmaster/dijnet-dump]
# Copyright (c) 2016-2021 <NAME> and others
# This software may be distributed under the terms of the Apache 2.0 license

if ! type xmllint wget xxd &>/dev/null; then
  echo "Dependency missing! Please install them:" >&2
  echo "- debian/ubuntu: apt-get install wget xxd libxml2-utils" >&2
  echo "- fedora: dnf install wget vim-common findutils" >&2
  echo "- openwrt: opkg install bash wget xxd libxml2-utils" >&2
  exit 1
fi

die() {
  EXIT_CODE="$?" && echo -e "$@" >&2 && exit 1
}

absolute_path() {
  pushd . >/dev/null && cd "$(dirname "$1")" && pwd -P && popd >/dev/null
}

xpath() {
  xmllint --html --xpath "$1" - 2>/dev/null
}

unaccent() {
  xxd -ps | tr -d '\n' | sed 's/\(..\)/ \1/g
    s/e1/61/g;s/c1/41/g;s/e9/65/g;s/c9/45/g;s/ed/69/g;s/cd/49/g;s/f3/6f/g;s/d3/4f/g;s/f5/6f/g
    s/d5/4f/g;s/f6/6f/g;s/d6/4f/g;s/fa/75/g;s/da/55/g;s/fb/75/g;s/db/55/g;s/fc/75/g;s/dc/55/g
    s/c3 a1/61/g;s/c3 81/41/g;s/c3 a9/65/g;s/c3 89/45/g;s/c3 ad/69/g;s/c3 8d/49/g
    s/c3 b3/6f/g;s/c3 93/4f/g;s/c3 b6/6f/g;s/c3 96/4f/g;s/c5 91/6f/g;s/c5 90/4f/g
    s/c3 ba/75/g;s/c3 9a/55/g;s/c3 bc/75/g;s/c3 9c/55/g;s/c5 b1/75/g;s/c5 b0/55/g' \
  | xxd -r -ps | sed 's/&\(.\)\(acute\|uml\|dblac\);/\1/g;s/&nbsp;/ /g'
}

dijnet() {
  URL_POSTFIX="$1"; shift; local IFS=""; POST_DATA="$*"
  wget --quiet --output-document=- --post-data "${POST_DATA}" --no-check-certificate \
       --load-cookies "${COOKIES}" --save-cookies "${COOKIES}" --keep-session-cookies \
       "${DIJNET_BASE_URL}/${URL_POSTFIX}"
}

invoice_data() {
  local IFS="|"; FILTER=$(sed 's/|/" or text()="/g' <<<"$*")
  unaccent <<<"${DOWNLOAD_PAGE}" | xpath '//label[text()="'"${FILTER}"'"]/../following-sibling::td[1]//text()'
}

download_internal_links() {
  LINKS=$(xpath '//a[contains(@class, "xt_link__download")]/@href' | sed 's/href="\([^"]*\)"/\1 /g')
  for LINK in ${LINKS}; do
    grep -q "^http" <<<"${LINK}" && continue
    wget --quiet --load-cookies "${COOKIES}" --content-disposition --no-clobber --no-check-certificate \
         --directory-prefix "${FIXED_TARGET_FOLDER}" "${DIJNET_BASE_URL}/ekonto/control/${LINK}"
  done
}

progress() {
  local PROVIDER_NAME=$(sed 's/ ()$//' <<<"${INVOICE_PROVIDER} (${INVOICE_PROVIDER_ALIAS})")
  if type pv &>/dev/null; then
    pv -N "download \"${PROVIDER_NAME}\", total: ${INVOICE_COUNT}, current" \
       -W -b -p -l -t -e -s "${INVOICE_COUNT}" >/dev/null
  else
    xargs -n 1 echo -ne "\033[2K\rdownload \"${PROVIDER_NAME}\", total: ${INVOICE_COUNT}, current:"; echo
  fi
}

set -o pipefail; export LANG=C LC_ALL=C
type gsed &>/dev/null && sed() { exec gsed "$@"; }
. "$(absolute_path "$0")/dijnet-dump.conf" || die "ERROR: config file (dijnet-dump.conf) missing"
[[ "$1" == "-d" ]] && DEBUG_LOG="yes" && shift
DIJNET_USERNAME="${1:-${DIJNET_USERNAME}}"
[[ -z "${DIJNET_USERNAME}" ]] && die "usage: $(basename "$0") [-d] username"
[[ -z "${DIJNET_PASSWORD}" ]] && read -r -s -p "password: " DIJNET_PASSWORD && echo
if [[ "${DEBUG_LOG}" == "yes" ]]; then
  exec 3> >(sed 's/'"${DIJNET_USERNAME}"'/<USERNAME>/g;s/'"${DIJNET_PASSWORD}"'/********/g' >dijnet-dump.log)
  export BASH_XTRACEFD="3"
  set -x
fi
{'fraction_non_alphanumeric': 0.2503088326127239, 'fraction_numerical': 0.04570722668313774, 'mean_word_length': 3.255584756898817, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 36, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2587353', 'n_tokens_mistral': 3128, 'n_tokens_neox': 2976, 'n_words': 562}