<commit_before>#!/bin/bash
set -eu
export JUJU_HOME=$HOME/cloud-city
source $JUJU_HOME/juju-qa.jujuci
set -x
revision_build=$1
WORKSPACE=$2
AGENT_JSON=$3
TESTING=$4
AGENT_JOBS=$5

WS_JSON=$WORKSPACE/ws-json
AGENT_DIRNAME=revision-build-$revision_build
WS_AGENTS=$WORKSPACE/agent/$AGENT_DIRNAME
TESTING_AGENTS=$TESTING/agent/$AGENT_DIRNAME
VERSION=$(jujuci.py get-build-vars $revision_build --version)
mkdir $WS_JSON
mkdir -p $WS_AGENTS
for job in $AGENT_JOBS; do
  jujuci.py get -b lastBuild $job '*.tgz' $WS_AGENTS
  jujuci.py get -b lastBuild $job '*.json' $WS_JSON
done
set_stream.py $AGENT_JSON/release.json \
  $WS_JSON/release-$revision_build.json $revision_build
mkdir -p $TESTING_AGENTS
cp $WS_AGENTS/* $TESTING_AGENTS
cp $WS_JSON/*.json $AGENT_JSON/
json2streams --juju-format $AGENT_JSON/* $TESTING
sstream-query $TESTING/streams/v1/index2.json \
  content_id="com.ubuntu.juju:revision-build-$revision_build:tools" \
  version=$VERSION --output-format="%(sha256)s  %(item_url)s" |sort|uniq > \
  sha256sums
sha256sum -c sha256sums<commit_msg>Get agents and json using s3ci.py<commit_after>#!/bin/bash
set -eu
export JUJU_HOME=${JUJU_HOME:-$HOME/cloud-city}
source $JUJU_HOME/juju-qa.jujuci
set -x
revision_build=$1
WORKSPACE=$2
AGENT_JSON=$3
TESTING=$4
AGENT_JOBS=$5

WS_JSON=$WORKSPACE/ws-json
AGENT_DIRNAME=revision-build-$revision_build
WS_AGENTS=$WORKSPACE/agent/$AGENT_DIRNAME
TESTING_AGENTS=$TESTING/agent/$AGENT_DIRNAME
VERSION=$(jujuci.py get-build-vars $revision_build --version)
mkdir $WS_JSON
mkdir -p $WS_AGENTS
for job in $AGENT_JOBS; do
  s3ci.py get $revision_build $job '.*\.tgz' $WS_AGENTS
  s3ci.py get $revision_build $job '.*\.json' $WS_JSON
done
set_stream.py $AGENT_JSON/release.json \
  $WS_JSON/release-$revision_build.json $revision_build
mkdir -p $TESTING_AGENTS
cp $WS_AGENTS/* $TESTING_AGENTS
cp $WS_JSON/*.json $AGENT_JSON/
json2streams --juju-format $AGENT_JSON/* $TESTING
sstream-query $TESTING/streams/v1/index2.json \
  content_id="com.ubuntu.juju:revision-build-$revision_build:tools" \
  version=$VERSION --output-format="%(sha256)s  %(item_url)s" |sort|uniq > \
  sha256sums
sha256sum -c sha256sums
{'fraction_non_alphanumeric': 0.13805970149253732, 'fraction_numerical': 0.02005597014925373, 'mean_word_length': 5.787974683544304, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '641964', 'n_tokens_mistral': 943, 'n_tokens_neox': 888, 'n_words': 152}