<commit_before>#!/bin/bash -xv

# Download zipcode file and reload

set -e

logdir=/var/log
logfile=${logdir}/$(basename $0).log
exec 2> ${logfile}

logger --stderr --priority user.info '***** get_kenall.sh *****'

homedir=/home/morinatsu/
dbdir=${homedir}/db/
bindir=${homedir}/bin/

rm -f ${dbdir}/ken_all.lzh
rm -f ${dbdir}/ken_all.csv
rm -f ${dbdir}/ken_all_utf8.csv
rm -f ${dbdir}/area.csv
rm -f ${dbdir}/net.csv

wget --quiet \
     --output-document=${dbdir}/ken_all.lzh \
     http://www.post.japanpost.jp/zipcode/dl/kogaki/lzh/ken_all.lzh
lha -ew=${dbdir} ${dbdir}/ken_all.lzh
nkf -SwdO ${dbdir}/ken_all.csv ${dbdir}/ken_all_utf8.csv
python ${bindir}/make_loaddata.py \
       ${dbdir}/ken_all_utf8.csv \
       ${dbdir}/area.csv ${dbdir}/net.csv
cd ${bindir}
sqlite3 ${dbdir}/zip_db < ${bindir}/zipcode.sql

exit 0<commit_msg>stoped ditribute of lzh archive<commit_after>#!/bin/bash -xv

# Download zipcode file and reload

set -e

logdir=/var/log
logfile=${logdir}/$(basename $0).log
exec 2> ${logfile}

logger --stderr --priority user.info '***** get_kenall.sh *****'

homedir=/home/morinatsu/
dbdir=${homedir}/db/
bindir=${homedir}/bin/

rm -f ${dbdir}/ken_all.lzh
rm -f ${dbdir}/ken_all.csv
rm -f ${dbdir}/ken_all_utf8.csv
rm -f ${dbdir}/area.csv
rm -f ${dbdir}/net.csv

wget --quiet \
     --output-document=${dbdir}/ken_all.zip \
     http://www.post.japanpost.jp/zipcode/dl/kogaki/zip/ken_all.zip
unzip ${dbdir}/ken_all.zip -d ${dbdir}/ken_all.csv
nkf -SwdO ${dbdir}/ken_all.csv ${dbdir}/ken_all_utf8.csv
python ${bindir}/make_loaddata.py \
       ${dbdir}/ken_all_utf8.csv \
       ${dbdir}/area.csv ${dbdir}/net.csv
cd ${bindir}
sqlite3 ${dbdir}/zip_db < ${bindir}/zipcode.sql

exit 0
{'fraction_non_alphanumeric': 0.2025836758661186, 'fraction_numerical': 0.008220786846741044, 'mean_word_length': 3.707182320441989, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4179826', 'n_tokens_mistral': 749, 'n_tokens_neox': 720, 'n_words': 126}