<commit_before>#!/usr/bin/env nix-shell
#!nix-shell -i bash -p git

set -eu

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

REMOTE="origin"
UPSTREAM="master"
TO_SHORT="16.09"
RELEASE_NAMED="release-${TO_SHORT}"

source "$DIR/state/port_state.sh"

readonly TMPTO=$(mktemp -d ported.XXXXXXXXXX -p "$TMPDIR")
cleanup() {
    rm -rf "$TMPTO"
}
trap cleanup EXIT


git fetch "$REMOTE"
git cherry "$REMOTE/$RELEASE_NAMED" "$REMOTE/$UPSTREAM" "$UPSTREAM_OLDEST" \
    | grep -v "+" \
    | cut -d' ' -f2 \
    | xargs git show \
    | git patch-id --stable > $TMPTO/upstream

git log --pretty=format:%H "$TO_OLDEST"..."$REMOTE/$RELEASE_NAMED" \
    | xargs git show \
    | git patch-id --stable > $TMPTO/against

cat <<EOF
From: <NAME> <<EMAIL>>
To: <EMAIL>
Subject: Security fixes from $(date -u "+%F %R %Z")
--text follows this line--
<#secure method=pgpmime mode=sign>

The following issues have been resolved in NixOS in unstable and
$RELEASE_NAMED. They remain potentially vulnerable on older major
releases.

These patches will be released to the unstable and
$RELEASE_NAMED channels when Hydra finishes building the "tested" job
for each channel:

 - https://hydra.nixos.org/job/nixos/${RELEASE_NAMED}/tested
 - https://hydra.nixos.org/job/nixos/trunk-combined/tested

Please consider helping with the next security roundup by commenting on
LATEST_ROUNDUP_URL.

EOF

table() {
  echo "$UPSTREAM|$TO_SHORT|Message|Notes"
  echo "---|---|---|---"
  for id in `cat $TMPTO/upstream | cut -d' ' -f1`; do
      master=$(git rev-parse --short $(cat $TMPTO/upstream | grep -G "^${id}" | cut -d' ' -f2))
      release=$(git rev-parse --short $(cat $TMPTO/against | grep -G "^${id}" | cut -d' ' -f2))
      message=$(git log --pretty=format:%s $master...$master~1)
      if [ $(echo "$message" | wc -c) -gt 50 ]; then
         message=$(echo -n "$message" | head -c47; echo "...");
      fi
      echo "$master|$release|$message|n/a"
  done
}

table | column -s"|" -t

update_state() {
    echo "TO_OLDEST=$(git rev-parse "$REMOTE/$RELEASE_NAMED")"
    echo "UPSTREAM_OLDEST=$(git rev-parse "$REMOTE/$UPSTREAM")"
}

update_state > "$DIR/state/port_state.sh"<commit_msg>Improve the ported.sh script to ... well, actually work.<commit_after>#!/usr/bin/env bash

#!/usr/bin/env nix-shell
#!nix-shell -i bash -p git

set -eu
set -o pipefail

TMPDIR="/tmp"
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

REMOTE="origin"
UPSTREAM="master"
TO_SHORT="16.09"
RELEASE_NAMED="release-${TO_SHORT}"

source "$DIR/state/port_state.sh"

readonly TMPTO=$(mktemp -d ported.XXXXXXXXXX -p "$TMPDIR")
cleanup() {
    rm -rf "$TMPTO"
}
trap cleanup EXIT

debug() {
    set +u
    if ! [ "x$DEBUG" = "x" ]; then
        echo "$@" >&2
    fi
    set -u
}

#git fetch "$REMOTE"

cherries() {
    debug "cherries: Fetching"
    git cherry "$REMOTE/$RELEASE_NAMED" "$REMOTE/$UPSTREAM" "$UPSTREAM_OLDEST"
    debug "cherries: fetched"
}

logs() {
    debug "logs: Fetching"
    git log --pretty=format:%H "$TO_OLDEST"..."$REMOTE/$RELEASE_NAMED"
    debug "logs: fetched"
}

cherries \
    | grep -v "+" \
    | cut -d' ' -f2 \
    | xargs git show \
    | git patch-id --stable > $TMPTO/upstream

logs \
    | xargs git show \
    | git patch-id --stable > $TMPTO/against

cat <<EOF
From: <NAME> <<EMAIL>>
To: <EMAIL>
Subject: Security fixes from $(date -u "+%F %R %Z")
--text follows this line--
<#secure method=pgpmime mode=sign>

The following issues have been resolved in NixOS in unstable and
$RELEASE_NAMED. They remain potentially vulnerable on older major
releases.

These patches will be released to the unstable and
$RELEASE_NAMED channels when Hydra finishes building the "tested" job
for each channel:

 - https://hydra.nixos.org/job/nixos/${RELEASE_NAMED}/tested
 - https://hydra.nixos.org/job/nixos/trunk-combined/tested

Please consider helping with the next security roundup by commenting on
LATEST_ROUNDUP_URL.

EOF

table() {
  for id in `cat $TMPTO/upstream | cut -d' ' -f1`; do
      debug "${id}: identify master hash"
      master_hash=$(cat $TMPTO/upstream | grep -G "^${id}" | cut -d' ' -f2 | tail -n1)
      debug "${id}: master -> $master_hash"

      set +e
      master_short=$(git rev-parse --short "${master_hash}")
      if [ "x" = "x$master_short" ]; then
          debug "${id}: BUG? Cannot find hash ${master_hash}"
          exit 1
      fi
      set -e

      debug "${id}: master -> $master_hash -> ${master_short}"

      debug "${id}: identify release hash"
      set +e
      release_hash=$(cat $TMPTO/against | grep -G "^${id}" | cut -d' ' -f2)
      if [ "x" = "x$release_hash" ]; then
          debug "${id}: did not identify candidate in release, skipping."
          continue
      fi
      set -e
      debug "${id}: release -> $release_hash"
      release_short=$(git rev-parse --short "${release_hash}")
      debug "${id}: master -> $master_hash -> ${release_short}"

      debug "${id}: fetch  message"
      message=$(git log --pretty=format:%s $master_hash...$master_hash~1)
      if [ $(echo "$message" | wc -c) -gt 50 ]; then
         message=$(echo -n "$message" | head -c47; echo "...");
      fi
      echo "$master_short|$release_short|$message|n/a" >> $TMPTO/table
      debug ""
  done

  echo "$UPSTREAM|$TO_SHORT|Message|Notes"
  echo "---|---|---|---"

  cat "$TMPTO/table" | sort -t\| -k3
}

table | column -s"|" -t

update_state() {
    echo "TO_OLDEST=$(git rev-parse "$REMOTE/$RELEASE_NAMED")"
    echo "UPSTREAM_OLDEST=$(git rev-parse "$REMOTE/$UPSTREAM")"
}

update_state > "$DIR/state/port_state.sh"
{'fraction_non_alphanumeric': 0.1735998559337295, 'fraction_numerical': 0.005762650819376914, 'mean_word_length': 3.0868285504047095, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 28, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2292742', 'n_tokens_mistral': 2103, 'n_tokens_neox': 1963, 'n_words': 551}