<commit_before>#!/usr/bin/env bash
set -e -o pipefail

if [ "0" -eq "$#" ]; then
  echo "Takes at least one argument, the list of packages to update." >&2
  exit 1
fi

cd "$(readlink -f "$0" | xargs dirname)"
source concurrent.lib.sh

TOP_LEVEL="$(git rev-parse --show-toplevel)"
cd $TOP_LEVEL

TMPDIR=""
cleanup() {
  CODE="$?"
  echo "Cleaning up..." >&2
  if [ -n "$TMPDIR" ]; then
    rm -rf "$TMPDIR"
  fi
  exit "$?"
}
trap cleanup EXIT ERR INT QUIT PIPE TERM
TMPDIR="$(mktemp -d)"
CONCURRENT_LOG_DIR="$TMPDIR/logs"


echo "Finding packages and all dependencies..." >&2
pkglist='[ '
for arg in "$@"; do
  pkglist+="\"$arg\" "
done
pkglist+=']'

nix-build --out-link $TMPDIR/nix-list --arg pkgList "$pkglist" -E '
  { pkgList }:
  let
    pkgs = (import pkgs/top-level/all-packages.nix { });
    allBuildInputs = pkg: pkg.buildInputs
      ++ pkg.nativeBuildInputs
      ++ pkg.propagatedBuildInputs
      ++ pkg.propagatedNativeBuildInputs;
    listPkgAndDeps = pkg:
      if pkg ? goPackagePath then
        pkgs.lib.foldl
          (attr: dep: attr // listPkgAndDeps dep)
          { "${pkg.goPackagePath}" = {
            inherit (pkg) rev;
            date = pkg.date or "nodate";
            name = let
              names = pkgs.lib.attrNames (pkgs.lib.filterAttrs
              (n: d: d ? goPackagePath && d.goPackagePath == pkg.goPackagePath) pkgs.goPackages);
            in if pkgs.lib.length names > 0 then pkgs.lib.head names else
              throw "Found no name for: ${pkg.goPackagePath}";
          }; }
          (allBuildInputs pkg)
      else { };
    combinedList = pkgs.lib.foldl
      (attr: n: attr // listPkgAndDeps pkgs.goPackages.${n})
      { }
      pkgList;
    pkgOutput = pkgs.lib.mapAttrsToList
      (n: d: "${n} ${d.rev} ${d.date} ${d.name}\n") combinedList;
  in
    pkgs.writeText "current-go-package-list" pkgOutput
'
cp $TMPDIR/nix-list $TMPDIR/list
rm $TMPDIR/nix-list


echo "Finding package repos..." >&2
build_redirect() {
  if echo "$1" | grep -q '\(github.com\|bitbucket.org\)'; then
    echo "$1" | sed 's,^[^h],https://\0,g'
    return 0
  fi

  local OUTPUT
  OUTPUT="$(curl "$1")"
  local IMPORT
  IMPORT="$(echo "$OUTPUT" | grep 'go-import' || true)"
  if [ -n "$IMPORT" ]; then
    echo "$IMPORT" | grep -q 'git'
    local REDIRECT
    REDIRECT="$(echo "$OUTPUT" | awk '{ if (/go-source/) { print $4 } }')"
    echo "$REDIRECT" >&2
    build_redirect "$REDIRECT"
  else
    echo "This site is unrecognized: $1" >&2<commit_msg>goUpdater: Allow disabling auto update<commit_after>#!/usr/bin/env bash
set -e -o pipefail

if [ "0" -eq "$#" ]; then
  echo "Takes at least one argument, the list of packages to update." >&2
  exit 1
fi

cd "$(readlink -f "$0" | xargs dirname)"
source concurrent.lib.sh

TOP_LEVEL="$(git rev-parse --show-toplevel)"
cd $TOP_LEVEL

TMPDIR=""
cleanup() {
  CODE="$?"
  echo "Cleaning up..." >&2
  if [ -n "$TMPDIR" ]; then
    rm -rf "$TMPDIR"
  fi
  exit "$?"
}
trap cleanup EXIT ERR INT QUIT PIPE TERM
TMPDIR="$(mktemp -d)"
CONCURRENT_LOG_DIR="$TMPDIR/logs"


echo "Finding packages and all dependencies..." >&2
pkglist='[ '
for arg in "$@"; do
  pkglist+="\"$arg\" "
done
pkglist+=']'

nix-build --out-link $TMPDIR/nix-list --arg pkgList "$pkglist" -E '
  { pkgList }:
  let
    pkgs = (import pkgs/top-level/all-packages.nix { });
    allBuildInputs = pkg: pkg.buildInputs
      ++ pkg.nativeBuildInputs
      ++ pkg.propagatedBuildInputs
      ++ pkg.propagatedNativeBuildInputs;
    listPkgAndDeps = pkg:
      if pkg ? goPackagePath then
        pkgs.lib.foldl
          (attr: dep: attr // listPkgAndDeps dep)
          { "${pkg.goPackagePath}" = {
            inherit (pkg) rev;
            date = pkg.date or "nodate";
            autoUpdate = pkg.autoUpdate or true;
            name = let
              names = pkgs.lib.attrNames (pkgs.lib.filterAttrs
              (n: d: d ? goPackagePath && d.goPackagePath == pkg.goPackagePath) pkgs.goPackages);
            in if pkgs.lib.length names > 0 then pkgs.lib.head names else
              throw "Found no name for: ${pkg.goPackagePath}";
          }; }
          (allBuildInputs pkg)
      else { };
    combinedList = pkgs.lib.foldl
      (attr: n: attr // listPkgAndDeps pkgs.goPackages.${n})
      { }
      pkgList;
    pkgOutput = pkgs.lib.mapAttrsToList
      (n: d: "${n} ${d.rev} ${d.date} ${d.name}\n")
      (pkgs.lib.filterAttrs (n: d: d.autoUpdate) combinedList);
  in
    pkgs.writeText "current-go-package-list" pkgOutput
'
cp $TMPDIR/nix-list $TMPDIR/list
rm $TMPDIR/nix-list


echo "Finding package repos..." >&2
build_redirect() {
  if echo "$1" | grep -q '\(github.com\|bitbucket.org\)'; then
    echo "$1" | sed 's,^[^h],https://\0,g'
    return 0
  fi

  local OUTPUT
  OUTPUT="$(curl "$1")"
  local IMPORT
  IMPORT="$(echo "$OUTPUT" | grep 'go-import' || true)"
  if [ -n "$IMPORT" ]; then
    echo "$IMPORT" | grep -q 'git'
    local REDIRECT
    REDIRECT="$(echo "$OUTPUT" | awk '{ if (/go-source/) { print $4 } }')"
    echo "$REDIRECT" >&2
    build_redirect "$REDIRECT"
  else
    echo "This site is unrecognized: $1" >&2
{'fraction_non_alphanumeric': 0.16404715127701375, 'fraction_numerical': 0.0066797642436149315, 'mean_word_length': 2.570126227208976, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '8413', 'n_tokens_mistral': 1935, 'n_tokens_neox': 1847, 'n_words': 485}