<commit_before>#!/bin/sh

# $0 url [url2 [url3 [...]]]
if [ -z "$1" ]; then
    exit;
fi


# external commands
AWK="/usr/bin/awk"
CUT="/usr/bin/cut"
GREP="/bin/grep"
MKDIR="/bin/mkdir"
TR="/usr/bin/tr"
WC="/usr/bin/wc"
WGET="/usr/bin/wget"

# -nv: basic option for simple message
# -4: some site has ipv6 address, but no route of ipv6, so force using ipv4 only
# --no-check-certificate: do not check ssl/cert for https:// url
WGETOPTION="-nv -4 -m --no-check-certificate"

# WGETUSERAGENT="Wget/1.12"  # default
WGETUSERAGENT="Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW; rv:1.9.2.24) Gecko/20111103 Firefox/3.6.24 (.NET CLR 3.5.30729)"  # pretend windows browsers

# referer default none
CLEANREFERER=1

# build tmp directory
${MKDIR} -p ~/tmp

cd ~/tmp

while [ ! -z "$1" ]; do
    if [ ${CLEANREFERER} -gt 0 ]; then
        WGETREFERER=""
    fi

    CLEANREFERER=0

    # avoid double-typed command
    if [ "wq" = "$1" -o "wq.sh" = "$1" -o "$0" = "$1" ]; then
        shift
        continue
    fi

    # check referer
    CHECKREFERER=`echo "$1" | ${CUT} -c 1-10`
    if [ "--referer=" = "${CHECKREFERER}" ]; then
        WGETREFERER="$1"
        shift
        continue
    fi

    # TODO: imgur.com
    # if url: http://imgur.com/Hwrk1Vl
    # get pic: http://i.imgur.com/Hwrk1Vl.jpg
    # check jpg from http://imgur.com/Hwrk1Vl content ?<commit_msg>update wq/wm.sh: replace wq to wm<commit_after>#!/bin/sh

# $0 url [url2 [url3 [...]]]
if [ -z "$1" ]; then
    exit;
fi


# external commands
AWK="/usr/bin/awk"
CUT="/usr/bin/cut"
GREP="/bin/grep"
MKDIR="/bin/mkdir"
TR="/usr/bin/tr"
WC="/usr/bin/wc"
WGET="/usr/bin/wget"

# -nv: basic option for simple message
# -4: some site has ipv6 address, but no route of ipv6, so force using ipv4 only
# --no-check-certificate: do not check ssl/cert for https:// url
WGETOPTION="-nv -4 -m --no-check-certificate"

# WGETUSERAGENT="Wget/1.12"  # default
WGETUSERAGENT="Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW; rv:1.9.2.24) Gecko/20111103 Firefox/3.6.24 (.NET CLR 3.5.30729)"  # pretend windows browsers

# referer default none
CLEANREFERER=1

# build tmp directory
${MKDIR} -p ~/tmp

cd ~/tmp

while [ ! -z "$1" ]; do
    if [ ${CLEANREFERER} -gt 0 ]; then
        WGETREFERER=""
    fi

    CLEANREFERER=0

    # avoid double-typed command
    if [ "wm" = "$1" -o "wm.sh" = "$1" -o "$0" = "$1" ]; then
        shift
        continue
    fi

    # check referer
    CHECKREFERER=`echo "$1" | ${CUT} -c 1-10`
    if [ "--referer=" = "${CHECKREFERER}" ]; then
        WGETREFERER="$1"
        shift
        continue
    fi

    # TODO: imgur.com
    # if url: http://imgur.com/Hwrk1Vl
    # get pic: http://i.imgur.com/Hwrk1Vl.jpg
    # check jpg from http://imgur.com/Hwrk1Vl content ?
{'fraction_non_alphanumeric': 0.17833698030634573, 'fraction_numerical': 0.040846097738876735, 'mean_word_length': 2.5623376623376624, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6008066', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1123, 'n_words': 285}