<commit_before>#!/bin/bash


# this script should be executed when box starts up and executed periodically
# this script itself should not depends on any partition space, meaning it should be able to run with all disks full
# threshold
# soft: (/log is over 85%)
# remove
# /var/log/*.gz
# /log/blog//files.gz
# /log/forever/main.log
# /log/forever/main_last.log
# any files under /log/firewalla that is older than one day
# hard (/log is over 95%)
# remove
# all files under soft
# /var/log/syslog
# /log/apt/cache/*
# /log/forever/*
# /log/firewalla/*




# ----------------------------------------------------------------------------
# Functions
# ----------------------------------------------------------------------------
mylog() {
    echo "$(date -u): $@"
}

loginfo() {
    mylog "INFO: $@"
}

logerror() {
    mylog "ERROR: $@" >&2
}

soft_clean() {
    loginfo "do SOFT cleaning ..."
    sudo rm -f /var/log/*.gz
    sudo rm -f /log/blog/*/files.*.gz
    rm -f /log/forever/main_last.log
    : > /log/forever/main.log
    # any files under /log/firewalla that is older than one day
    find /log/firewalla -mtime +1 -delete
}

hard_clean() {
    loginfo "do HARD cleaning ..."
    : | sudo  tee /var/log/syslog
    sudo rm -rf /log/apt/cache/*
    sudo chown pi:pi /log/forever/*
    ls /log/forever/*.log | xargs -r -I FILE sh -c ': > FILE'
    ls /log/firewalla/*.log | xargs -r -I FILE sh -c ': > FILE'
}

# ----------------------------------------------------------------------------
# MAIN goes here
# ----------------------------------------------------------------------------

use_percent=$( df --output=pcent /log | tail -1 | tr -d ' %' )<commit_msg>cleanup /log/blog/*/*.gz in hard clean<commit_after>#!/bin/bash


# this script should be executed when box starts up and executed periodically
# this script itself should not depends on any partition space, meaning it should be able to run with all disks full
# threshold
# soft: (/log is over 85%)
# remove
# /var/log/*.gz
# /log/blog//files.gz
# /log/forever/main.log
# /log/forever/main_last.log
# any files under /log/firewalla that is older than one day
# hard (/log is over 95%)
# remove
# all files under soft
# /var/log/syslog
# /log/apt/cache/*
# /log/forever/*
# /log/firewalla/*




# ----------------------------------------------------------------------------
# Functions
# ----------------------------------------------------------------------------
mylog() {
    echo "$(date -u): $@"
}

loginfo() {
    mylog "INFO: $@"
}

logerror() {
    mylog "ERROR: $@" >&2
}

soft_clean() {
    loginfo "do SOFT cleaning ..."
    sudo rm -f /var/log/*.gz
    sudo rm -f /log/blog/*/files.*.gz
    rm -f /log/forever/main_last.log
    : > /log/forever/main.log
    # any files under /log/firewalla that is older than one day
    find /log/firewalla -mtime +1 -delete
}

hard_clean() {
    loginfo "do HARD cleaning ..."
    : | sudo  tee /var/log/syslog
    sudo rm -rf /log/apt/cache/*
    sudo rm -f /log/blog/*/*.gz
    sudo chown pi:pi /log/forever/*
    ls /log/forever/*.log | xargs -r -I FILE sh -c ': > FILE'
    ls /log/firewalla/*.log | xargs -r -I FILE sh -c ': > FILE'
}

# ----------------------------------------------------------------------------
# MAIN goes here
# ----------------------------------------------------------------------------

use_percent=$( df --output=pcent /log | tail -1 | tr -d ' %' )
{'fraction_non_alphanumeric': 0.3117456579334707, 'fraction_numerical': 0.004121283485428319, 'mean_word_length': 1.3997175141242937, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7017713', 'n_tokens_mistral': 1105, 'n_tokens_neox': 1047, 'n_words': 324}