<commit_before>#!/usr/bin/env bash

#
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

set -eo pipefail

CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"

# Need to grabs source for MPL, GPL, LGPL, and CDDL licenced dependencies
# and include it in the Docker image

# This should be reviewed before each release to make sure we're capturing all
# the dependencies we need.

rm dependencies-src.zip || true

dependencies=("slog-json")

zip="$(pwd)/dependencies-src.zip"
pushd "$CARGO_HOME/registry/src"
for d in "${dependencies[@]}"; do
  path=$(find . -type d -name "$d-*")
  echo "Archiving $d:$path"
  zip -rv "$zip" "$path"
done
popd<commit_msg>Fix bug archiving dependencies (#442)<commit_after>#!/usr/bin/env bash

#
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

set -eo pipefail

CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"

# Need to grabs source for MPL, GPL, LGPL, and CDDL licenced dependencies
# and include it in the Docker image

# This should be reviewed before each release to make sure we're capturing all
# the dependencies we need.

rm dependencies-src.zip || true

dependencies=("slog-json")

zip="$(pwd)/dependencies-src.zip"

echo "Archiving dependencies to: $zip"
pushd "$CARGO_HOME/registry/src"
for d in "${dependencies[@]}"; do
  find . -type d -name "$d-*" | xargs -I {} zip -rv "$zip" "{}"
done
popd
{'fraction_non_alphanumeric': 0.10590205106739221, 'fraction_numerical': 0.00795311845960653, 'mean_word_length': 4.139784946236559, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1855162', 'n_tokens_mistral': 740, 'n_tokens_neox': 696, 'n_words': 306}