<commit_before>#!/bin/sh

SOURCE_DIR="$1"
DOCSET_DIR="$2/RustNightly.docset"
CONTENT_DIR="$DOCSET_DIR/Contents"
DOC_DIR="$DOCSET_DIR/Contents/Resources/Documents"
IDX_DIR="$DOCSET_DIR/Contents/Resources"
mkdir -p $DOC_DIR && cp -R $SOURCE_DIR/ $DOC_DIR && cp Info.plist $CONTENT_DIR/ && (find $DOC_DIR -type f -name \*.html | ./indexer.py $DOC_DIR $IDX_DIR)

# tar --exclude='.DS_Store' -cvzf <docset name>.tgz <docset name>.docset
# sha1
# update xml version
# upload to S3 xml + tgz
# check permissions
# dash-feed://https%3A%2F%2Fs3-us-west-2.amazonaws.com%2Fnet.vhbit.rust-doc%2FRustNightly.xml<commit_msg>Automated packing and meta info gen<commit_after>#!/bin/sh

SOURCE_DIR="$1"
OUT_DIR="$2"
DOCSET_NAME="RustNightly.docset"
DOCSET_TGZ="RustNightly.tgz"
DOCSET_DIR="$OUT_DIR/$DOCSET_NAME"
CONTENT_DIR="$DOCSET_DIR/Contents"
DOC_DIR="$DOCSET_DIR/Contents/Resources/Documents"
IDX_DIR="$DOCSET_DIR/Contents/Resources"
mkdir -p "$DOC_DIR"
echo "Copying"
cp -R "$SOURCE_DIR/" "$DOC_DIR"
cp Info.plist "$CONTENT_DIR/"
cp icon.png $OUT_DIR/$DOCSET_NAME/
echo "Indexing"
find $DOC_DIR -type f -name \*.html | ./indexer.py $DOC_DIR $IDX_DIR
cp RustNightly.xml $OUT_DIR
cd $OUT_DIR
echo "Packing"
tar --exclude='.DS_Store' -czf $DOCSET_TGZ $DOCSET_NAME
echo "Metadata"
HASH=`openssl sha1 $DOCSET_TGZ | cut -d " " -f 2`
DT=`date +%Y-%m-%d`
cat ../RustNightly.xml | sed "s/__DT__/$DT/g;s/__SHA__/$HASH/g" > RustNightly.xml

# upload to S3 xml + tgz
# check permissions
# dash-feed://https%3A%2F%2Fs3-us-west-2.amazonaws.com%2Fnet.vhbit.rust-doc%2FRustNightly.xml

# pages with refresh

{'fraction_non_alphanumeric': 0.1657179000632511, 'fraction_numerical': 0.01454775458570525, 'mean_word_length': 5.379032258064516, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2378370', 'n_tokens_mistral': 707, 'n_tokens_neox': 649, 'n_words': 122}