<commit_before>#!/bin/bash

# Crawl internal file system for changes by creating a catalog of diffs.
# Usage:
# Make the scripts executable: chmod +x rev*sh
# Set the $target after the exec script like this to catalog /tmp
# ./revexec.sh /tmp
# Then visit /var/low/crawl for your data.

# Import the "strip" from my CryptoCore .bashrc but as stripdown

stripdown () {
tr -d '\040\011\012\015'
}

mkdir /var/log/crawl 2> /dev/null
mkdir /var/log/crawl/low 2> /dev/null
chown 755 /var/log/crawl/low
diffdir=$(pwd | cut -f2-999)

DATESTAMP=$(date | stripdown )
cd /
chmod 777 /var/log/crawl/*

function larthdiff {
diffdir=$(pwd | cut -c2-999)
difffile="data""$DATESTAMP"
echo /var/log/crawl/low"$diffdir" | xargs mkdir -p
touch /var/log/crawl/low"$diffdir"/"$difffile".larth.prev
touch /var/log/crawl/low"$diffdir"/"$difffile".larth
cp /var/log/crawl/low"$diffdir"/"$difffile".larth /var/log/crawl/low"$diffdir"/"$difffile".larth.prev
touch /var/log/crawl/low"$diffdir"/"$difffile".larth
date >> /var/log/crawl/low"$diffdir"/"$difffile".larth
ls -larth >> /var/log/crawl/low"$diffdir"/"$difffile".larth
diff /var/log/crawl/low"$diffdir"/"$difffile".larth /var/log/crawl/low"$diffdir"/"$difffile".larth.prev > /var/log/crawl/low"$diffdir"/"$difffile"."$DATESTAMP"diff
cat /var/log/crawl/low"$diffdir"/"$difffile"."$DATESTAMP"diff >> /var/log/crawldiff.log
}

function lsdirs {
touch /var/log/crawl/low"$diffdir"/"$difffile".ls
date >> /var/log/crawl/low"$diffdir"/"$difffile".ls
ls > /var/log/crawl/low"$diffdir"/"$difffile".ls
}
larthdiff
lsdirs

# Crawl the system in a loop.

function runit {
    larthdiff;
    lsdirs;
}

# Clean up.
cp /dev/null /var/log/crawl/.pwd
# Main loop.
for target in "$@"; do
find "$target"  >> /var/log/crawl/.pwd
runit
done

for y in $(cat /var/log/crawl/.pwd); do
    cd /"$y"
    runit 
done
<commit_msg>naming and redirect corrections<commit_after>#!/bin/bash

# Crawl internal file system for changes.



# Import the "strip" from my CryptoCore .bashrc but as stripdown

stripdown () {
tr -d '\040\011\012\015'
}

mkdir /var/log/crawl 2> /dev/null
mkdir /var/log/crawl/low 2> /dev/null
chown 755 /var/log/crawl/low
diffdir=$(pwd | cut -f2-999)

DATESTAMP=$(date | stripdown )
cd /
chmod 777 /var/log/crawl/*

function larthdiff {
diffdir=$(pwd | cut -c2-999)
echo /var/log/crawl/low"$diffdir" | xargs mkdir -p
touch /var/log/crawl/low"$diffdir"/data.larth.prev
touch /var/log/crawl/low"$diffdir"/data.larth
cp /var/log/crawl/low"$diffdir"/data.larth /var/log/crawl/low"$diffdir"/data.larth.prev
touch /var/log/crawl/low"$diffdir"/data.larth
date >> /var/log/crawl/low"$diffdir"/data.larth
ls -larth >> /var/log/crawl/low"$diffdir"/data.larth
diff /var/log/crawl/low"$diffdir"/data.larth /var/log/crawl/low"$diffdir"/data.larth.prev > /var/log/crawl/low"$diffdir"/data."$DATESTAMP"diff
cat /var/log/crawl/low"$diffdir"/data."$DATESTAMP"diff >> /var/log/crawldiff.log
}

function lsdirs {
touch /var/log/crawl/low"$diffdir"/data.ls
date > /var/log/crawl/low"$diffdir"/data.ls
ls > /var/log/crawl/low"$diffdir"/data.ls
}
larthdiff
lsdirs

# Crawl the system in a loop.

function runit {
    larthdiff;
    lsdirs;
}

# Clean up.
cp /dev/null /var/log/crawl/.pwd
# Main loop.
for target in "$@"; do
find "$target"  >> /var/log/crawl/.pwd
runit
done

for y in $(cat /var/log/crawl/.pwd); do
    cd /"$y"
    runit
done

{'fraction_non_alphanumeric': 0.18339307048984468, 'fraction_numerical': 0.016726403823178016, 'mean_word_length': 3.7436260623229463, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1935327', 'n_tokens_mistral': 1557, 'n_tokens_neox': 1438, 'n_words': 274}