<commit_before>CACHE_DIR=cache
EN_FULL=${CACHE_DIR}/en_full.txt
WORD_LIST=word_list.txt
DICTIONARY_SIZE?=8192

all: ${WORD_LIST}
	head -n ${DICTIONARY_SIZE} word_list.txt | ./generate.sh

${WORD_LIST}: ${EN_FULL}
	cat ${EN_FULL} \
	  | perl -pe 's# [0-9]+$$##' \
	  | (iconv -f UTF-8 -t ASCII -c; true) \
	  > ${WORD_LIST}

${EN_FULL}: ${CACHE_DIR}
	curl https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2016/en/en_full.txt > ${EN_FULL}

${CACHE_DIR}:
	mkdir -p ${CACHE_DIR}

clean:
	rm -rf ${CACHE_DIR} ${WORD_LIST}
<commit_msg>remove duplicate lines in the fetched word list<commit_after>CACHE_DIR=cache
EN_FULL=${CACHE_DIR}/en_full.txt
WORD_LIST=word_list.txt
DICTIONARY_SIZE?=8192

all: ${WORD_LIST}
	head -n ${DICTIONARY_SIZE} word_list.txt | ./generate.sh

${WORD_LIST}: ${EN_FULL}
	cat ${EN_FULL} \
	  | perl -pe 's# [0-9]+$$##' \
	  | (iconv -f UTF-8 -t ASCII -c; true) \
	  | perl -ne '$$H{$$_}++ or print' \
	  > ${WORD_LIST}

${EN_FULL}: ${CACHE_DIR}
	curl https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2016/en/en_full.txt > ${EN_FULL}

${CACHE_DIR}:
	mkdir -p ${CACHE_DIR}

clean:
	rm -rf ${CACHE_DIR} ${WORD_LIST}

{'fraction_non_alphanumeric': 0.188663282571912, 'fraction_numerical': 0.018612521150592216, 'mean_word_length': 4.426605504587156, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1125222', 'n_tokens_mistral': 529, 'n_tokens_neox': 497, 'n_words': 94}