<commit_before>#!/bin/sh -e
#
# Update our copy of the MCL site's database snapshots
#
# 2016-08-24 - created
#
# Copyright 2016 California Institute of Technology. ALL RIGHTS
# RESERVED. U.S. Government Sponsorship acknowledged.


# Please keep these secret:
username='USERNAME'
password='PASSWORD'


# Where we store stuff:
filestorage='/var/www/html/static/mcl-portal-snapshots/filestorage'
blobstorage='/var/www/html/static/mcl-portal-snapshots/blobstorage'


# First the content:
[ -d "$filestorage" ] || mkdir --parents "$filestorage"
cd "$filestorage"
/usr/bin/wget \
    --quiet \
    --execute robots=off \
    --timestamping \
    --no-check-certificate \
    --user="$username" \
    --password="$password" \
    https://mcl.nci.nih.gov/var/filestorage/Data.fs


# Now the blobs:
[ -d "$blobstorage" ] || mkdir --parents "$blobstorage"
cd "$blobstorage"
/usr/bin/wget \
    --quiet \
    --execute robots=off \
    --cut-dirs=1 \
    --reject='index.html*' \
    --no-host-directories \
    --mirror \
    --no-parent \
    --relative \
    --timestamping \
    --no-check-certificate \
    --recursive \
    --user="$username" \
    --password="$password" \
    https://edrn.nci.nih.gov/blobstorage/


# Yay:
exit 0<commit_msg>use var<commit_after>#!/bin/sh -e
#
# Update our copy of the MCL site's database snapshots
#
# 2016-08-24 - created
#
# Copyright 2016 California Institute of Technology. ALL RIGHTS
# RESERVED. U.S. Government Sponsorship acknowledged.


# Please keep these secret:
username='USERNAME'
password='PASSWORD'


# Where we store stuff:
filestorage='/var/www/html/static/mcl-portal-snapshots/filestorage'
blobstorage='/var/www/html/static/mcl-portal-snapshots/blobstorage'


# First the content:
[ -d "$filestorage" ] || mkdir --parents "$filestorage"
cd "$filestorage"
/usr/bin/wget \
    --quiet \
    --execute robots=off \
    --timestamping \
    --no-check-certificate \
    --user="$username" \
    --password="<PASSWORD>" \
    https://mcl.nci.nih.gov/var/filestorage/Data.fs


# Now the blobs:
[ -d "$blobstorage" ] || mkdir --parents "$blobstorage"
cd "$blobstorage"
/usr/bin/wget \
    --quiet \
    --execute robots=off \
    --cut-dirs=1 \
    --reject='index.html*' \
    --no-host-directories \
    --mirror \
    --no-parent \
    --relative \
    --timestamping \
    --no-check-certificate \
    --recursive \
    --user="$username" \
    --password="<PASSWORD>" \
    https://mcl.nci.nih.gov/var/blobstorage/


# Yay:
exit 0
{'fraction_non_alphanumeric': 0.16996366572466692, 'fraction_numerical': 0.011303996770286637, 'mean_word_length': 2.606986899563319, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1513912', 'n_tokens_mistral': 912, 'n_tokens_neox': 863, 'n_words': 172}