<commit_before>
clean_js:
	## to make real tests, we need to dump a non-time-sensitive
	## extract from these Chunk files and compare it to a stored
	## copy of what is expected.
	git checkout -- data/john-smith/john-smith-tagged-by-lingpipe-0.sc
	rm -f data/john-smith/john-smith-tagged-by-lingpipe-test-0.sc
	rm -rf tmp

john-smith: clean_js clean install
	echo data/john-smith/original | python -m kba.pipeline.run configs/john-smith.yaml
	echo data/john-smith/original | python -m kba.pipeline.run configs/john-smith-lingpipe.yaml
	echo data/john-smith/john-smith-0.sc | python -m kba.pipeline.run configs/john-smith-lingpipe-from-chunk.yaml

	## figure out how to make tests from this
	##diff data/john-smith/john-smith-tagged-by-lingpipe-0.sc  data/john-smith/john-smith-tagged-by-lingpipe-test-0.sc

clean: clean_js
	rm -rf build dist src/kba.pipeline.egg-info

install: clean<commit_msg>updating Makefile to install libxml2-dev when possible; also cleaning up the "clean" target a bit more.<commit_after>
clean_js:
	## to make real tests, we need to dump a non-time-sensitive
	## extract from these Chunk files and compare it to a stored
	## copy of what is expected.
	git checkout -- data/john-smith/john-smith-0.sc
	git checkout -- data/john-smith/john-smith-tagged-by-lingpipe-0.sc
	rm -f data/john-smith/john-smith-tagged-by-lingpipe-test-0.sc
	rm -rf tmp

john-smith: clean_js clean install
	echo data/john-smith/original | python -m kba.pipeline.run configs/john-smith.yaml
	echo data/john-smith/original | python -m kba.pipeline.run configs/john-smith-lingpipe.yaml
	echo data/john-smith/john-smith-0.sc | python -m kba.pipeline.run configs/john-smith-lingpipe-from-chunk.yaml

	## figure out how to make tests from this
	##diff data/john-smith/john-smith-tagged-by-lingpipe-0.sc  data/john-smith/john-smith-tagged-by-lingpipe-test-0.sc

clean: clean_js
	rm -rf build dist src/kba.pipeline.egg-info

.IGNORE: lxml
lxml:
	## this should be done by cloudinit/puppet or something along
	## those lines
	sudo apt-get -y install libxml2-dev libxslt-dev  

install: clean lxml
{'fraction_non_alphanumeric': 0.11366906474820145, 'fraction_numerical': 0.006235011990407674, 'mean_word_length': 3.7194570135746607, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '385536', 'n_tokens_mistral': 775, 'n_tokens_neox': 728, 'n_words': 196}