<commit_before>#!/usr/bin/env bash
cd "$(dirname "$(readlink -f "$0")")"

cleanup() {
  excode=$?
  rm -rf "$TMPDIR"
  trap - EXIT
  exit $excode
}
TMPDIR="$(mktemp -d)"
trap cleanup EXIT HUP INT QUIT PIPE TERM

CONCURRENT_LOG_DIR=$TMPDIR/logs
source concurrent.lib.sh

TOPDIR="$(pwd)"
while [ ! -d "$TOPDIR/pkgs" ]; do
  TOPDIR="$(dirname "$TOPDIR")"
done

HASHES=($(grep -r '"Qm' "$TOPDIR/pkgs" | sed 's,.*"\(Qm[a-zA-Z0-9]*\)".*,\1,g'))

fetch() {
  ipfs pin add "$1"
}

ARGS=()
for HASH in "${HASHES[@]}"; do
  ARGS+=("-" "$HASH" "fetch" "$HASH")
done
concurrent "${ARGS[@]}"<commit_msg>pin-to-ipfs: Improve performance<commit_after>#!/usr/bin/env bash
cd "$(dirname "$(readlink -f "$0")")"

cleanup() {
  excode=$?
  rm -rf "$TMPDIR"
  trap - EXIT
  exit $excode
}
TMPDIR="$(mktemp -d)"
trap cleanup EXIT HUP INT QUIT PIPE TERM

CONCURRENT_LOG_DIR=$TMPDIR/logs
source concurrent.lib.sh

TOPDIR="$(pwd)"
while [ ! -d "$TOPDIR/pkgs" ]; do
  TOPDIR="$(dirname "$TOPDIR")"
done

HASHES=($(grep -r '"Qm' "$TOPDIR/pkgs" | sed 's,.*"\(Qm[a-zA-Z0-9]*\)".*,\1,g' | sort | uniq))

declare -A current
while read h; do
  current["$h"]=1
done < <(ipfs pin ls -t recursive | awk '{print $1}' | sort | uniq)

fetch() {
  ipfs pin add "$1"
}

ARGS=()
for HASH in "${HASHES[@]}"; do
  if [ "${current[$HASH]}" != "1" ]; then
    ARGS+=("-" "$HASH" "fetch" "$HASH")
  fi
done
if [ "${#ARGS[@]}" -gt "0" ]; then
  concurrent "${ARGS[@]}"
fi
{'fraction_non_alphanumeric': 0.24982456140350878, 'fraction_numerical': 0.009824561403508772, 'mean_word_length': 3.986013986013986, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1538103', 'n_tokens_mistral': 645, 'n_tokens_neox': 617, 'n_words': 155}